send preloaded CODAP document info to LLM

concord-consortium · Jan 7, 2025 · 728cdc3 · 728cdc3
1 parent 9d6cdc1
commit 728cdc3
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 10 deletions.
diff --git a/src/components/App.tsx b/src/components/App.tsx
@@ -1,15 +1,15 @@
-import React, { useEffect, useState } from "react";
+import React, { useCallback, useEffect, useRef, useState } from "react";
 import { observer } from "mobx-react-lite";
-import { initializePlugin, selectSelf } from "@concord-consortium/codap-plugin-api";
+import { addDataContextChangeListener, addDataContextsListListener, ClientNotification, getDataContext, getListOfDataContexts, initializePlugin, selectSelf } from "@concord-consortium/codap-plugin-api";
 import { useAppConfigContext } from "../hooks/use-app-config-context";
 import { useAssistantStore } from "../hooks/use-assistant-store";
 import { ChatInputComponent } from "./chat-input";
 import { ChatTranscriptComponent } from "./chat-transcript";
 import { ReadAloudMenu } from "./readaloud-menu";
 import { KeyboardShortcutControls } from "./keyboard-shortcut-controls";
-import { DAVAI_SPEAKER, GREETING, USER_SPEAKER } from "../constants";
+import { DAVAI_SPEAKER, DEBUG_SPEAKER, GREETING, USER_SPEAKER } from "../constants";
 import { DeveloperOptionsComponent } from "./developer-options";
-import { getUrlParam } from "../utils/utils";
+import { formatJsonMessage, getUrlParam } from "../utils/utils";
 
 import "./App.scss";
 
@@ -30,15 +30,40 @@ export const App = observer(() => {
   const modeUrlParam = getUrlParam("mode") || "";
   const isDevMode = modeUrlParam === "development" || appConfig.mode === "development";
   const [showDebugLog, setShowDebugLog] = useState(isDevMode);
+  const assistantStoreRef = useRef(assistantStore);
+
+  const handleDocumentChangeNotice = useCallback(async (notification: ClientNotification) => {
+    // ignore the dataContextCountChanged notification; it's sent when a data context is added or removed,
+    // along with another notification that provides the actual details of the change
+    if (notification.values.operation === "dataContextCountChanged") return;
+    assistantStoreRef.current.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Document change notice", content: formatJsonMessage(notification)});
+    await assistantStoreRef.current.sendCODAPDataContexts();
+  }, []);
+
+  const handleDataContextChangeNotice = useCallback(async (notification: ClientNotification) => {
+    assistantStoreRef.current.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Data context change notice", content: formatJsonMessage(notification)});
+    // resource is in the form of "dataContextChangeNotice[<dataContextName>]";
+    // unfortunately the dataContext name isn't otherwise available in the notification object
+    const dataCtxName = notification.resource.replace("dataContextChangeNotice[", "").replace("]", "");
+    const updatedDataContext = await getDataContext(dataCtxName);
+    await assistantStoreRef.current.sendCODAPDocumentInfo(`Data context ${dataCtxName} has been updated: ${JSON.stringify(updatedDataContext.values)}`);
+  }, []);
 
   useEffect(() => {
-    initializePlugin({pluginName: kPluginName, version: kVersion, dimensions});
+    const init = async () => {
+      await initializePlugin({pluginName: kPluginName, version: kVersion, dimensions});
+      addDataContextsListListener(handleDocumentChangeNotice);
+      const dataContexts = await getListOfDataContexts();
+      dataContexts.values.forEach((ctx: Record<string, any>) => addDataContextChangeListener(ctx.name, handleDataContextChangeNotice));
+    };
+    init();
     selectSelf();
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
   useEffect(() => {
     assistantStore.initializeAssistant();
+    assistantStoreRef.current = assistantStore;
   }, [assistantStore, appConfig.assistantId]);
 
   const handleFocusShortcut = () => {

diff --git a/src/models/assistant-model.ts b/src/models/assistant-model.ts
@@ -1,7 +1,7 @@
 import { types, flow, Instance } from "mobx-state-tree";
 import { OpenAI } from "openai";
 import { Message } from "openai/resources/beta/threads/messages";
-import { codapInterface } from "@concord-consortium/codap-plugin-api";
+import { codapInterface, getDataContext, getListOfDataContexts } from "@concord-consortium/codap-plugin-api";
 import { DAVAI_SPEAKER, DEBUG_SPEAKER } from "../constants";
 import { convertBase64ToImage, formatJsonMessage } from "../utils/utils";
 import { requestThreadDeletion } from "../utils/openai-utils";
@@ -80,6 +80,7 @@ export const AssistantModel = types
           description: "New thread created",
           content: formatJsonMessage(self.thread)
         });
+        sendCODAPDataContexts();
       } catch (err) {
         console.error("Failed to initialize assistant:", err);
         self.transcriptStore.addMessage(DEBUG_SPEAKER, {
@@ -89,14 +90,46 @@ export const AssistantModel = types
       }
     });
 
+    const sendCODAPDataContexts = flow(function* () {
+      try {
+        const contexts = yield getListOfDataContexts();
+        const contextsDetails: Record<string, any> = {};
+        for (const ctx of contexts.values) {
+          const { name } = ctx;
+          const ctxDetails = yield getDataContext(name);
+          contextsDetails[name] = ctxDetails.values;
+        }
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Data contexts information", content: formatJsonMessage(contextsDetails)});
+        sendCODAPDocumentInfo(`Data contexts: ${JSON.stringify(contextsDetails)}`);
+      } catch (err) {
+        console.error("Failed to get data contexts:", err);
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Failed to get data contexts", content: formatJsonMessage(err)});
+      }
+    });
+
+    const sendCODAPDocumentInfo = flow(function* (message) {
+      try {
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Sending CODAP document information to LLM", content: message});
+        const messageSent = yield self.apiConnection.beta.threads.messages.create(self.thread.id, {
+          role: "user",
+          content: `This is a system message containing information about the CODAP document. ${message}`,
+        });
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "CODAP document information received by LLM", content: formatJsonMessage(messageSent)});
+      }
+      catch (err) {
+        console.error("Failed to send system message:", err);
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Failed to send CODAP document information to LLM", content: formatJsonMessage(err)});
+      }
+    });
+
     const handleMessageSubmit = flow(function* (messageText) {
       try {
         const messageSent = yield self.apiConnection.beta.threads.messages.create(self.thread.id, {
           role: "user",
           content: messageText,
         });
         self.isLoadingResponse = true;
-        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Message sent to LLM", content: formatJsonMessage(messageSent)});
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Message received by LLM", content: formatJsonMessage(messageSent)});
         yield startRun();
 
       } catch (err) {
@@ -259,11 +292,12 @@ export const AssistantModel = types
           ))
           : [];
 
-        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Tool outputs", content: formatJsonMessage(toolOutputs)});
+        self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Tool outputs being submitted", content: formatJsonMessage(toolOutputs)});
         if (toolOutputs) {
-          yield self.apiConnection.beta.threads.runs.submitToolOutputs(
+          const submittedToolOutputsRes = yield self.apiConnection.beta.threads.runs.submitToolOutputs(
             self.thread.id, runId, { tool_outputs: toolOutputs }
           );
+          self.transcriptStore.addMessage(DEBUG_SPEAKER, {description: "Tool outputs received", content: formatJsonMessage(submittedToolOutputsRes)});
         }
       } catch (err) {
         console.error(err);
@@ -301,7 +335,7 @@ export const AssistantModel = types
       }
     });
 
-    return { createThread, deleteThread, initializeAssistant, handleMessageSubmit };
+    return { createThread, deleteThread, initializeAssistant, handleMessageSubmit, sendCODAPDataContexts, sendCODAPDocumentInfo };
   });
 
 export interface AssistantModelType extends Instance<typeof AssistantModel> {}