Merge branch 'scratch-package-refactor-merge' into tagged-template-do…

…cumentation
mitmedialab · Jul 24, 2024 · f7942d4 · f7942d4
2 parents a711fe3 + 8e110f5
commit f7942d4
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 31 deletions.
diff --git a/extensions/src/poseBody/package.json b/extensions/src/poseBody/package.json
@@ -11,6 +11,6 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@tensorflow-models/posenet": "^2.2.1"
+    "@tensorflow-models/posenet": "2.2.1"
   }
 }
diff --git a/extensions/src/poseBody/pnpm-lock.yaml b/extensions/src/poseBody/pnpm-lock.yaml
diff --git a/extensions/src/selfieSegmentation/index.ts b/extensions/src/selfieSegmentation/index.ts
@@ -5,7 +5,8 @@ import type BlockUtility from "$scratch-vm/engine/block-utility";
 
 const details: ExtensionMenuDisplayDetails = {
   name: "Selfie Detector",
-  tags: ["Made by PRG"]
+  tags: ["Made by PRG"],
+  insetIconURL: "profile-picture.png",
 };
 
 export default class extends extension(

diff --git a/extensions/src/selfieSegmentation/profile-picture.png b/extensions/src/selfieSegmentation/profile-picture.png
diff --git a/extensions/src/teachableMachine/index.ts b/extensions/src/teachableMachine/index.ts
@@ -25,7 +25,7 @@ export default class teachableMachine extends extension({
   iconURL: "teachable-machine-blocks.png",
   insetIconURL: "teachable-machine-blocks-small.svg",
   tags: ["Dancing with AI", "Made by PRG"]
-}) {
+}, "indicators") {
   lastUpdate: number;
   maxConfidence: number;
   modelConfidences: {};
@@ -166,18 +166,24 @@ export default class teachableMachine extends extension({
   }
 
   async startPredicting(modelDataUrl) {
-    if (!this.predictionState[modelDataUrl]) {
-      try {
-        this.predictionState[modelDataUrl] = {};
-        // https://github.com/googlecreativelab/teachablemachine-community/tree/master/libraries/image
-        const { model, type } = await this.initModel(modelDataUrl);
-        this.predictionState[modelDataUrl].modelType = type;
-        this.predictionState[modelDataUrl].model = model;
-        this.runtime.requestToolboxExtensionsUpdate();
-      } catch (e) {
-        this.predictionState[modelDataUrl] = {};
-        console.log("Model initialization failure!", e);
-      }
+    const alreadyLoaded = Boolean(this.predictionState[modelDataUrl]);
+    try {
+      const indicator = await this.indicate({
+        type: "warning",
+        msg: alreadyLoaded ? "Updating model" : "Loading model"
+      });
+      this.predictionState[modelDataUrl] = {};
+      // https://github.com/googlecreativelab/teachablemachine-community/tree/master/libraries/image
+      const { model, type } = await this.initModel(modelDataUrl);
+      this.predictionState[modelDataUrl].modelType = type;
+      this.predictionState[modelDataUrl].model = model;
+      this.runtime.requestToolboxExtensionsUpdate();
+      indicator.close();
+      this.indicateFor({ type: "success", msg: "Model loaded" }, 1);
+    } catch (e) {
+      this.predictionState[modelDataUrl] = {};
+      console.log("Model initialization failure!", e);
+      this.indicateFor({ type: "error", msg: "Unable to load model." }, 1);
     }
   }
 
@@ -195,8 +201,9 @@ export default class teachableMachine extends extension({
   }
 
   async initModel(modelUrl) {
-    const modelURL = modelUrl + "model.json";
-    const metadataURL = modelUrl + "metadata.json";
+    const avoidCache = `?x=${Date.now()}`;
+    const modelURL = modelUrl + "model.json" + avoidCache;
+    const metadataURL = modelUrl + "metadata.json" + avoidCache;
     const customMobileNet = await tmImage.load(modelURL, metadataURL);
     if ((customMobileNet as any)._metadata.hasOwnProperty('tfjsSpeechCommandsVersion')) {
       // customMobileNet.dispose(); // too early to dispose
@@ -217,24 +224,29 @@ export default class teachableMachine extends extension({
       const customPoseNet = await tmPose.load(modelURL, metadataURL);
       return { model: customPoseNet, type: this.ModelType.POSE };
     } else {
+      console.log(customMobileNet.getMetadata(), customMobileNet.getTotalClasses(), customMobileNet.getClassLabels());
       return { model: customMobileNet, type: this.ModelType.IMAGE };
     }
   }
 
   useModel(url) {
     try {
       const modelUrl = this.modelArgumentToURL(url);
-      this.getPredictionStateOrStartPredicting(modelUrl);
+      this.getPredictionStateOrStartPredicting(modelUrl, true);
       this.updateStageModel(modelUrl);
     } catch (e) {
       this.teachableImageModel = null;
     }
   }
 
-  modelArgumentToURL(modelArg) {
-    return modelArg.startsWith('https://teachablemachine.withgoogle.com/models/') ?
-      modelArg :
-      `https://teachablemachine.withgoogle.com/models/${modelArg}/`;
+  modelArgumentToURL(modelArg: string) {
+    const endpointProvidedFromInterface = "https://teachablemachine.withgoogle.com/models/";
+    // NOTE: It's possible Google will change this endpoint in the future, and that will break this extension.
+    // TODO: https://github.com/mitmedialab/prg-extension-boilerplate/issues/343
+    const redirectEndpoint = "https://storage.googleapis.com/tm-model/";
+    return modelArg.startsWith(endpointProvidedFromInterface)
+      ? modelArg.replace(endpointProvidedFromInterface, redirectEndpoint)
+      : redirectEndpoint + modelArg + "/";
   }
 
   updateStageModel(modelUrl) {
@@ -245,9 +257,9 @@ export default class teachableMachine extends extension({
     }
   }
 
-  getPredictionStateOrStartPredicting(modelUrl) {
+  getPredictionStateOrStartPredicting(modelUrl, override = false) {
     const hasPredictionState = this.predictionState.hasOwnProperty(modelUrl);
-    if (!hasPredictionState) {
+    if (!hasPredictionState || override) {
       this.startPredicting(modelUrl);
       return null;
     }