Merge branch 'main' into add-provider-visibility-to-node

# Conflicts:
#	core/src/stores/postgres.rs

core/src/stores/postgres.rs

@@ -167,14 +167,15 @@ impl PostgresStore {
         let stmt = tx
             .prepare(
                 "INSERT INTO data_sources_nodes \
-                  (id, data_source, created, node_id, timestamp, title, mime_type, provider_visibility, parents, \
+                  (id, data_source, created, node_id, timestamp, title, mime_type, provider_visibility, parents, source_url, \
                    document, \"table\", folder) \
-                  VALUES (DEFAULT, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) \
+                  VALUES (DEFAULT, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \
                   ON CONFLICT (data_source, node_id) DO UPDATE \
                   SET timestamp = EXCLUDED.timestamp, title = EXCLUDED.title, \
                     mime_type = EXCLUDED.mime_type, parents = EXCLUDED.parents, \
                     document = EXCLUDED.document, \"table\" = EXCLUDED.\"table\", \
-                    folder = EXCLUDED.folder, provider_visibility = EXCLUDED.provider_visibility \
+                    folder = EXCLUDED.folder, source_url = EXCLUDED.source_url \
+                    provider_visibility = EXCLUDED.provider_visibility \
                   RETURNING id",
             )
             .await?;
@@ -191,6 +192,7 @@ impl PostgresStore {
                     &upsert_params.mime_type,
                     &upsert_params.provider_visibility,
                     &upsert_params.parents,
+                    &upsert_params.source_url,
                     &document_row_id,
                     &table_row_id,
                     &folder_row_id,

front/admin/cli.ts

@@ -1,18 +1,7 @@
-import {
-  assertNever,
-  ConnectorsAPI,
-  removeNulls,
-  SUPPORTED_MODEL_CONFIGS,
-} from "@dust-tt/types";
-import parseArgs from "minimist";
-
 import { getConversation } from "@app/lib/api/assistant/conversation";
 import { renderConversationForModel } from "@app/lib/api/assistant/generation";
-import {
-  getTextContentFromMessage,
-  getTextRepresentationFromMessages,
-} from "@app/lib/api/assistant/utils";
-import config from "@app/lib/api/config";
+import { getTextRepresentationFromMessages } from "@app/lib/api/assistant/utils";
+import { default as config } from "@app/lib/api/config";
 import { getDataSources } from "@app/lib/api/data_sources";
 import { garbageCollectGoogleDriveDocument } from "@app/lib/api/poke/plugins/data_sources/garbage_collect_google_drive_document";
 import { Authenticator } from "@app/lib/auth";
@@ -38,6 +27,14 @@ import {
   stopRetrieveTranscriptsWorkflow,
 } from "@app/temporal/labs/client";
 import { REGISTERED_CHECKS } from "@app/temporal/production_checks/activities";
+import { DustAPI } from "@dust-tt/client";
+import {
+  assertNever,
+  ConnectorsAPI,
+  removeNulls,
+  SUPPORTED_MODEL_CONFIGS,
+} from "@dust-tt/types";
+import parseArgs from "minimist";
 
 // `cli` takes an object type and a command as first two arguments and then a list of arguments.
 const workspace = async (command: string, args: parseArgs.ParsedArgs) => {
@@ -493,6 +490,45 @@ const productionCheck = async (command: string, args: parseArgs.ParsedArgs) => {
       );
       return;
     }
+    case "check-apps": {
+      if (!args.url) {
+        throw new Error("Missing --url argument");
+      }
+      if (!args.wId) {
+        throw new Error("Missing --wId argument");
+      }
+      if (!args.spaceId) {
+        throw new Error("Missing --spaceId argument");
+      }
+      const api = new DustAPI(
+        config.getDustAPIConfig(),
+        { apiKey: args.apiKey, workspaceId: args.wId },
+        logger,
+        args.url
+      );
+
+      const actions = Object.values(DustProdActionRegistry);
+
+      const res = await api.checkApps(
+        {
+          apps: actions.map((action) => ({
+            appId: action.app.appId,
+            appHash: action.app.appHash,
+          })),
+        },
+        args.spaceId
+      );
+      if (res.isErr()) {
+        throw new Error(res.error.message);
+      }
+      const notDeployedApps = res.value.filter((a) => !a.deployed);
+      if (notDeployedApps.length > 0) {
+        throw new Error(
+          "Missing apps: " + notDeployedApps.map((a) => a.appId).join(", ")
+        );
+      }
+      console.log("All apps are deployed");
+    }
   }
 };
 

front/components/data_source/MultipleDocumentsUpload.tsx

@@ -122,7 +122,7 @@ export const MultipleDocumentsUpload = ({
             // Have to use the filename to avoid fileId becoming apparent in the UI.
             upsertArgs: {
               title: blob.filename,
-              name: blob.filename,
+              document_id: blob.filename,
             },
           });
 

front/components/spaces/SpaceAppsList.tsx

@@ -8,19 +8,32 @@ import {
   Spinner,
   usePaginationFromUrl,
 } from "@dust-tt/sparkle";
-import type { ConnectorType, SpaceType, WorkspaceType } from "@dust-tt/types";
+import type {
+  AppType,
+  ConnectorType,
+  LightWorkspaceType,
+  SpaceType,
+  WorkspaceType,
+} from "@dust-tt/types";
 import type { CellContext } from "@tanstack/react-table";
 import { sortBy } from "lodash";
+import Link from "next/link";
 import { useRouter } from "next/router";
 import type { ComponentType } from "react";
-import { useRef } from "react";
-import { useState } from "react";
 import * as React from "react";
+import { useRef, useState } from "react";
 
 import { SpaceCreateAppModal } from "@app/components/spaces/SpaceCreateAppModal";
-import { useApps } from "@app/lib/swr/apps";
+import type { Action } from "@app/lib/registry";
+import {
+  DustProdActionRegistry,
+  PRODUCTION_DUST_APPS_SPACE_ID,
+  PRODUCTION_DUST_APPS_WORKSPACE_ID,
+} from "@app/lib/registry";
+import { useApps, useSavedRunStatus } from "@app/lib/swr/apps";
 
 type RowData = {
+  app: AppType;
   category: string;
   name: string;
   icon: ComponentType;
@@ -44,10 +57,85 @@ const getTableColumns = () => {
   ];
 };
 
+const getDustAppsColumns = (owner: WorkspaceType) => ({
+  id: "hash",
+  cell: (info: CellContext<RowData, string>) => {
+    const { app } = info.row.original;
+    const registryApp = Object.values(DustProdActionRegistry).find(
+      (action) => action.app.appId === app.sId
+    );
+    if (!registryApp) {
+      return (
+        <DataTable.CellContent>
+          <span>No registry app</span>
+        </DataTable.CellContent>
+      );
+    }
+    return (
+      <DataTable.CellContent>
+        <AppHashChecker owner={owner} app={app} registryApp={registryApp.app} />
+      </DataTable.CellContent>
+    );
+  },
+  accessorFn: (row: RowData) => row.name,
+});
+
+type AppHashCheckerProps = {
+  owner: LightWorkspaceType;
+  app: AppType;
+  registryApp: Action["app"];
+};
+
+const AppHashChecker = ({ owner, app, registryApp }: AppHashCheckerProps) => {
+  const { run, isRunError } = useSavedRunStatus(owner, app, (data) => {
+    switch (data?.run?.status?.run) {
+      case "running":
+        return 100;
+      default:
+        return 0;
+    }
+  });
+
+  if (
+    registryApp.appHash &&
+    run?.app_hash &&
+    registryApp.appHash !== run.app_hash
+  ) {
+    return (
+      <span>
+        Inconsistent hashes,{" "}
+        <Link
+          className="text-blue-500"
+          href={`/w/${owner.sId}/spaces/${app.space.sId}/apps/${app.sId}/specification?hash=${registryApp.appHash}`}
+          onClick={(e) => {
+            e.stopPropagation();
+          }}
+        >
+          compare
+        </Link>
+      </span>
+    );
+  }
+
+  if (isRunError) {
+    return <span>Error: {isRunError.error?.message}</span>;
+  }
+
+  if (!run) {
+    return <span>No run found</span>;
+  }
+
+  if (run?.status.run === "errored") {
+    return <span>Run failed</span>;
+  }
+
+  return "";
+};
+
 interface SpaceAppsListProps {
   canWriteInSpace: boolean;
   onSelect: (sId: string) => void;
-  owner: WorkspaceType;
+  owner: LightWorkspaceType;
   space: SpaceType;
 }
 
@@ -73,6 +161,7 @@ export const SpaceAppsList = ({
   const rows: RowData[] = React.useMemo(
     () =>
       sortBy(apps, "name").map((app) => ({
+        app,
         sId: app.sId,
         category: "apps",
         name: app.name,
@@ -91,6 +180,14 @@ export const SpaceAppsList = ({
     );
   }
 
+  const columns = getTableColumns();
+  if (
+    owner.sId === PRODUCTION_DUST_APPS_WORKSPACE_ID &&
+    space.sId === PRODUCTION_DUST_APPS_SPACE_ID
+  ) {
+    columns.push(getDustAppsColumns(owner));
+  }
+
   return (
     <>
       {rows.length === 0 ? (
@@ -140,7 +237,7 @@ export const SpaceAppsList = ({
           </div>
           <DataTable
             data={rows}
-            columns={getTableColumns()}
+            columns={columns}
             className="pb-4"
             filter={appSearch}
             filterColumn="name"

front/lib/api/config.ts

@@ -161,6 +161,26 @@ const config = {
   getStatusPageApiToken: (): string => {
     return EnvironmentConfig.getEnvVariable("STATUS_PAGE_API_TOKEN");
   },
+  getDustAppsSyncEnabled: (): boolean => {
+    return (
+      EnvironmentConfig.getOptionalEnvVariable("DUST_APPS_SYNC_ENABLED") ===
+      "true"
+    );
+  },
+  getDustAppsSyncMasterApiUrl: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_URL");
+  },
+  getDustAppsSyncMasterWorkspaceId: (): string => {
+    return EnvironmentConfig.getEnvVariable(
+      "DUST_APPS_SYNC_MASTER_WORKSPACE_ID"
+    );
+  },
+  getDustAppsSyncMasterSpaceId: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_SPACE_ID");
+  },
+  getDustAppsSyncMasterApiKey: (): string => {
+    return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_KEY");
+  },
 };
 
 export default config;