[front] Import/export apps api (#9870)

* Import/export apps * use system keys * more flexible schema * Add read permission check * Skip app run if no dataset * extract method to lib * Use poke for triggering sync * swagger * missing files * missing files * more logging/error info * Add possibility to display previous specification stored in core * Update front/lib/api/poke/plugins/spaces/sync_apps.ts Co-authored-by: Flavien David <flavien.david74@gmail.com> * Update front/lib/api/poke/plugins/spaces/sync_apps.ts Co-authored-by: Flavien David <flavien.david74@gmail.com> * Update front/lib/utils/apps.ts Co-authored-by: Flavien David <flavien.david74@gmail.com> * review comments * split function * doc * replaced promise.all * lint * Add column with dust-app status * review comments * Track and return app run errors * cleaning results * Add app checks script * check endpoint * Update front/components/spaces/SpaceAppsList.tsx Co-authored-by: Flavien David <flavien.david74@gmail.com> * review comments * format --------- Co-authored-by: Flavien David <flavien.david74@gmail.com>
dust-tt · Jan 14, 2025 · 1e206b1 · 1e206b1
1 parent cab4e8b
commit 1e206b1
Show file tree

Hide file tree

Showing 14 changed files with 948 additions and 24 deletions.
diff --git a/front/admin/cli.ts b/front/admin/cli.ts
@@ -1,18 +1,7 @@
-import {
-  assertNever,
-  ConnectorsAPI,
-  removeNulls,
-  SUPPORTED_MODEL_CONFIGS,
-} from "@dust-tt/types";
-import parseArgs from "minimist";
-
 import { getConversation } from "@app/lib/api/assistant/conversation";
 import { renderConversationForModel } from "@app/lib/api/assistant/generation";
-import {
-  getTextContentFromMessage,
-  getTextRepresentationFromMessages,
-} from "@app/lib/api/assistant/utils";
-import config from "@app/lib/api/config";
+import { getTextRepresentationFromMessages } from "@app/lib/api/assistant/utils";
+import { default as config } from "@app/lib/api/config";
 import { getDataSources } from "@app/lib/api/data_sources";
 import { garbageCollectGoogleDriveDocument } from "@app/lib/api/poke/plugins/data_sources/garbage_collect_google_drive_document";
 import { Authenticator } from "@app/lib/auth";
@@ -38,6 +27,14 @@ import {
   stopRetrieveTranscriptsWorkflow,
 } from "@app/temporal/labs/client";
 import { REGISTERED_CHECKS } from "@app/temporal/production_checks/activities";
+import { DustAPI } from "@dust-tt/client";
+import {
+  assertNever,
+  ConnectorsAPI,
+  removeNulls,
+  SUPPORTED_MODEL_CONFIGS,
+} from "@dust-tt/types";
+import parseArgs from "minimist";
 
 // `cli` takes an object type and a command as first two arguments and then a list of arguments.
 const workspace = async (command: string, args: parseArgs.ParsedArgs) => {
@@ -493,6 +490,45 @@ const productionCheck = async (command: string, args: parseArgs.ParsedArgs) => {
       );
       return;
     }
+    case "check-apps": {
+      if (!args.url) {
+        throw new Error("Missing --url argument");
+      }
+      if (!args.wId) {
+        throw new Error("Missing --wId argument");
+      }
+      if (!args.spaceId) {
+        throw new Error("Missing --spaceId argument");
+      }
+      const api = new DustAPI(
+        config.getDustAPIConfig(),
+        { apiKey: args.apiKey, workspaceId: args.wId },
+        logger,
+        args.url
+      );
+
+      const actions = Object.values(DustProdActionRegistry);
+
+      const res = await api.checkApps(
+        {
+          apps: actions.map((action) => ({
+            appId: action.app.appId,
+            appHash: action.app.appHash,
+          })),
+        },
+        args.spaceId
+      );
+      if (res.isErr()) {
+        throw new Error(res.error.message);
+      }
+      const notDeployedApps = res.value.filter((a) => !a.deployed);
+      if (notDeployedApps.length > 0) {
+        throw new Error(
+          "Missing apps: " + notDeployedApps.map((a) => a.appId).join(", ")
+        );
+      }
+      console.log("All apps are deployed");
+    }
   }
 };
 

diff --git a/front/components/spaces/SpaceAppsList.tsx b/front/components/spaces/SpaceAppsList.tsx
@@ -8,19 +8,32 @@ import {
   Spinner,
   usePaginationFromUrl,
 } from "@dust-tt/sparkle";
-import type { ConnectorType, SpaceType, WorkspaceType } from "@dust-tt/types";
+import type {
+  AppType,
+  ConnectorType,
+  LightWorkspaceType,
+  SpaceType,
+  WorkspaceType,
+} from "@dust-tt/types";
 import type { CellContext } from "@tanstack/react-table";
 import { sortBy } from "lodash";
+import Link from "next/link";
 import { useRouter } from "next/router";
 import type { ComponentType } from "react";
-import { useRef } from "react";
-import { useState } from "react";
 import * as React from "react";
+import { useRef, useState } from "react";
 
 import { SpaceCreateAppModal } from "@app/components/spaces/SpaceCreateAppModal";
-import { useApps } from "@app/lib/swr/apps";
+import type { Action } from "@app/lib/registry";
+import {
+  DustProdActionRegistry,
+  PRODUCTION_DUST_APPS_SPACE_ID,
+  PRODUCTION_DUST_APPS_WORKSPACE_ID,
+} from "@app/lib/registry";
+import { useApps, useSavedRunStatus } from "@app/lib/swr/apps";
 
 type RowData = {
+  app: AppType;
   category: string;
   name: string;
   icon: ComponentType;
@@ -44,10 +57,85 @@ const getTableColumns = () => {
   ];
 };
 
+const getDustAppsColumns = (owner: WorkspaceType) => ({
+  id: "hash",
+  cell: (info: CellContext<RowData, string>) => {
+    const { app } = info.row.original;
+    const registryApp = Object.values(DustProdActionRegistry).find(
+      (action) => action.app.appId === app.sId
+    );
+    if (!registryApp) {
+      return (
+        <DataTable.CellContent>
+          <span>No registry app</span>
+        </DataTable.CellContent>
+      );
+    }
+    return (
+      <DataTable.CellContent>
+        <AppHashChecker owner={owner} app={app} registryApp={registryApp.app} />
+      </DataTable.CellContent>
+    );
+  },
+  accessorFn: (row: RowData) => row.name,
+});
+
+type AppHashCheckerProps = {
+  owner: LightWorkspaceType;
+  app: AppType;
+  registryApp: Action["app"];
+};
+
+const AppHashChecker = ({ owner, app, registryApp }: AppHashCheckerProps) => {
+  const { run, isRunError } = useSavedRunStatus(owner, app, (data) => {
+    switch (data?.run?.status?.run) {
+      case "running":
+        return 100;
+      default:
+        return 0;
+    }
+  });
+
+  if (
+    registryApp.appHash &&
+    run?.app_hash &&
+    registryApp.appHash !== run.app_hash
+  ) {
+    return (
+      <span>
+        Inconsistent hashes,{" "}
+        <Link
+          className="text-blue-500"
+          href={`/w/${owner.sId}/spaces/${app.space.sId}/apps/${app.sId}/specification?hash=${registryApp.appHash}`}
+          onClick={(e) => {
+            e.stopPropagation();
+          }}
+        >
+          compare
+        </Link>
+      </span>
+    );
+  }
+
+  if (isRunError) {
+    return <span>Error: {isRunError.error?.message}</span>;
+  }
+
+  if (!run) {
+    return <span>No run found</span>;
+  }
+
+  if (run?.status.run === "errored") {
+    return <span>Run failed</span>;
+  }
+
+  return "";
+};
+
 interface SpaceAppsListProps {
   canWriteInSpace: boolean;
   onSelect: (sId: string) => void;
-  owner: WorkspaceType;
+  owner: LightWorkspaceType;
   space: SpaceType;
 }
 
@@ -73,6 +161,7 @@ export const SpaceAppsList = ({
   const rows: RowData[] = React.useMemo(
     () =>
       sortBy(apps, "name").map((app) => ({
+        app,
         sId: app.sId,
         category: "apps",
         name: app.name,
@@ -91,6 +180,14 @@ export const SpaceAppsList = ({
     );
   }
 
+  const columns = getTableColumns();
+  if (
+    owner.sId === PRODUCTION_DUST_APPS_WORKSPACE_ID &&
+    space.sId === PRODUCTION_DUST_APPS_SPACE_ID
+  ) {
+    columns.push(getDustAppsColumns(owner));
+  }
+
   return (
     <>
       {rows.length === 0 ? (
@@ -140,7 +237,7 @@ export const SpaceAppsList = ({
           </div>
           <DataTable
             data={rows}
-            columns={getTableColumns()}
+            columns={columns}
             className="pb-4"
             filter={appSearch}
             filterColumn="name"

diff --git a/front/lib/api/config.ts b/front/lib/api/config.ts
@@ -161,6 +161,26 @@ const config = {
   getStatusPageApiToken: (): string => {
     return EnvironmentConfig.getEnvVariable("STATUS_PAGE_API_TOKEN");
   },
+  getDustAppsSyncEnabled: (): boolean => {
+    return (
+      EnvironmentConfig.getOptionalEnvVariable("DUST_APPS_SYNC_ENABLED") ===
+      "true"
+    );
+  },
+  getDustAppsSyncMasterApiUrl: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_URL");
+  },
+  getDustAppsSyncMasterWorkspaceId: (): string => {
+    return EnvironmentConfig.getEnvVariable(
+      "DUST_APPS_SYNC_MASTER_WORKSPACE_ID"
+    );
+  },
+  getDustAppsSyncMasterSpaceId: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_SPACE_ID");
+  },
+  getDustAppsSyncMasterApiKey: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_KEY");
+  },
 };
 
 export default config;
diff --git a/front/lib/api/poke/plugins/index.ts b/front/lib/api/poke/plugins/index.ts
@@ -1,4 +1,5 @@
 export * from "./data_source_views";
 export * from "./data_sources";
 export * from "./global";
+export * from "./spaces";
 export * from "./workspaces";
diff --git a/front/lib/api/poke/plugins/spaces/index.ts b/front/lib/api/poke/plugins/spaces/index.ts
@@ -0,0 +1 @@
+export * from "./sync_apps";
diff --git a/front/lib/api/poke/plugins/spaces/sync_apps.ts b/front/lib/api/poke/plugins/spaces/sync_apps.ts
@@ -0,0 +1,40 @@
+import { Err, Ok } from "@dust-tt/types";
+
+import { createPlugin } from "@app/lib/api/poke/types";
+import { SpaceResource } from "@app/lib/resources/space_resource";
+import { synchronizeDustApps } from "@app/lib/utils/apps";
+
+export const syncAppsPlugin = createPlugin(
+  {
+    id: "sync-apps",
+    name: "Sync dust-apps",
+    description: "Synchronize dust-apps from production",
+    resourceTypes: ["spaces"],
+    args: {},
+  },
+  async (auth, spaceId) => {
+    if (!spaceId) {
+      return new Err(new Error("No space specified"));
+    }
+
+    const space = await SpaceResource.fetchById(auth, spaceId);
+    if (!space) {
+      return new Err(new Error("Space not found"));
+    }
+    const result = await synchronizeDustApps(auth, space);
+    if (result.isErr()) {
+      return new Err(new Error(`Error when syncing: ${result.error.message}`));
+    }
+    if (!result.value) {
+      return new Ok({
+        display: "text",
+        value: "Sync not enabled.",
+      });
+    }
+
+    return new Ok({
+      display: "json",
+      value: { importedApp: result.value },
+    });
+  }
+);