add shadow read and content nodes comparison

dust-tt · Jan 17, 2025 · 77dc6f3 · 77dc6f3
1 parent 99d7604
commit 77dc6f3
Show file tree

Hide file tree

Showing 2 changed files with 89 additions and 0 deletions.
diff --git a/front/lib/api/content_nodes.ts b/front/lib/api/content_nodes.ts
@@ -1,6 +1,7 @@
 import type {
   ContentNodeType,
   CoreAPIContentNode,
+  DataSourceViewContentNode,
   DataSourceViewType,
 } from "@dust-tt/types";
 import {
@@ -12,6 +13,7 @@ import {
 } from "@dust-tt/types";
 
 import type { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource";
+import type logger from "@app/logger/logger";
 
 export function getContentNodeInternalIdFromTableId(
   dataSourceView: DataSourceViewResource | DataSourceViewType,
@@ -49,6 +51,57 @@ export function getContentNodeInternalIdFromTableId(
   }
 }
 
+export function compareNodes({
+  connectorsContentNodes,
+  coreContentNodes,
+  localLogger,
+}: {
+  connectorsContentNodes: DataSourceViewContentNode[];
+  coreContentNodes: DataSourceViewContentNode[];
+  localLogger: typeof logger;
+}) {
+  connectorsContentNodes.forEach((connectorsNode) => {
+    const coreNodes = coreContentNodes.filter(
+      (coreNode) => coreNode.internalId === connectorsNode.internalId
+    );
+    if (coreNodes.length !== 1) {
+      localLogger.info(
+        {
+          internalId: connectorsNode.internalId,
+          coreNodesId: coreNodes.map((n) => n.internalId),
+        },
+        "[CoreNodes] Invalid match"
+      );
+    } else {
+      const coreNode = coreNodes[0];
+      const diff = Object.fromEntries(
+        Object.entries(connectorsNode)
+          .filter(
+            ([key, value]) =>
+              value !== coreNode[key as keyof DataSourceViewContentNode]
+          )
+          .map(([key, value]) => [
+            key,
+            {
+              connectors: value,
+              core: coreNode[key as keyof DataSourceViewContentNode],
+            },
+          ])
+      );
+
+      if (Object.keys(diff).length > 0) {
+        localLogger.info(
+          {
+            internalId: connectorsNode.internalId,
+            diff,
+          },
+          "[CoreNodes] Node mismatch"
+        );
+      }
+    }
+  });
+}
+
 export function getContentNodeMetadata(
   node: CoreAPIContentNode,
   viewType: "tables" | "documents"

diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts
@@ -12,6 +12,7 @@ import assert from "assert";
 
 import config from "@app/lib/api/config";
 import {
+  compareNodes,
   getContentNodeInternalIdFromTableId,
   getContentNodeMetadata,
 } from "@app/lib/api/content_nodes";
@@ -343,6 +344,41 @@ export async function getContentNodesForDataSourceView(
     }
 
     contentNodesResult = contentNodesRes.value;
+
+    const localLogger = logger.child({
+      dataSourceId: dataSourceView.dataSource.sId,
+      dataSourceViewId: dataSourceView.sId,
+      provider: dataSourceView.dataSource.connectorProvider,
+    });
+
+    // shadow read from core
+    const coreContentNodesRes =
+      await getContentNodesForManagedDataSourceViewFromCore(
+        dataSourceView,
+        params
+      );
+
+    if (contentNodesRes.isErr()) {
+      localLogger.info(
+        { error: contentNodesRes.error },
+        "[CoreNodes] Could not fetch content nodes from core"
+      );
+    } else if (coreContentNodesRes.isOk()) {
+      if (coreContentNodesRes.value.total !== contentNodesResult.total) {
+        localLogger.info(
+          {
+            coreNodesCount: coreContentNodesRes.value.total,
+            connectorsNodesCount: contentNodesResult.total,
+          },
+          "[CoreNodes] Content nodes count mismatch"
+        );
+      }
+      compareNodes({
+        connectorsContentNodes: contentNodesResult.nodes,
+        coreContentNodes: coreContentNodesRes.value.nodes,
+        localLogger,
+      });
+    }
   } else {
     const contentNodesRes = await getContentNodesForStaticDataSourceView(
       dataSourceView,