Merge branch 'main' into thomas/anthropic-adjust

dust-tt · Jan 14, 2025 · 8b1f84d · 8b1f84d
2 parents 75795c2 + 1e206b1
commit 8b1f84d
Show file tree

Hide file tree

Showing 103 changed files with 2,241 additions and 1,131 deletions.
diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts
@@ -1,4 +1,4 @@
-import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
+import { MIME_TYPES } from "@dust-tt/types";
 import { makeScript } from "scripts/helpers";
 
 import {
@@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) {
         parents: [getTeamsInternalId(connector.id)],
         parentId: null,
         title: "Conversations",
-        mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS,
+        mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER,
       });
     }
 
@@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [teamInternalId, getTeamsInternalId(connector.id)],
             parentId: getTeamsInternalId(connector.id),
             title: team.name,
-            mimeType: INTERCOM_MIME_TYPES.TEAM,
+            mimeType: MIME_TYPES.INTERCOM.TEAM,
           });
         }
       },
@@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [helpCenterInternalId],
             parentId: null,
             title: helpCenter.name,
-            mimeType: INTERCOM_MIME_TYPES.HELP_CENTER,
+            mimeType: MIME_TYPES.INTERCOM.HELP_CENTER,
           });
         }
 
@@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) {
                 parents: collectionParents,
                 parentId: collectionParents[1] || null,
                 title: collection.name,
-                mimeType: INTERCOM_MIME_TYPES.COLLECTION,
+                mimeType: MIME_TYPES.INTERCOM.COLLECTION,
               });
             }
           },

diff --git a/connectors/src/api/get_content_node_parents.ts b/connectors/src/api/get_content_node_parents.ts
diff --git a/connectors/src/api_server.ts b/connectors/src/api_server.ts
@@ -13,7 +13,6 @@ import {
   getConnectorsAPIHandler,
 } from "@connectors/api/get_connector";
 import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions";
-import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents";
 import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes";
 import { pauseConnectorAPIHandler } from "@connectors/api/pause_connector";
 import { resumeConnectorAPIHandler } from "@connectors/api/resume_connector";
@@ -112,11 +111,6 @@ export function startServer(port: number) {
     "/connectors/:connector_id/permissions",
     getConnectorPermissionsAPIHandler
   );
-  app.post(
-    // must be POST because of body
-    "/connectors/:connector_id/content_nodes/parents",
-    getContentNodesParentsAPIHandler
-  );
   app.post(
     // must be POST because of body
     "/connectors/:connector_id/content_nodes",

diff --git a/connectors/src/connectors/confluence/lib/permissions.ts b/connectors/src/connectors/confluence/lib/permissions.ts
@@ -53,7 +53,6 @@ export function createContentNodeFromSpace(
     sourceUrl: `${baseUrl}/wiki${urlSuffix}`,
     expandable: isExpandable,
     permission,
-    dustDocumentId: null,
     lastUpdatedAt: null,
   };
 }
@@ -75,7 +74,6 @@ export function createContentNodeFromPage(
     sourceUrl: `${baseUrl}/wiki${page.externalUrl}`,
     expandable: isExpandable,
     permission: "read",
-    dustDocumentId: null,
     lastUpdatedAt: null,
   };
 }

diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts
@@ -1,8 +1,8 @@
 import type { ModelId } from "@dust-tt/types";
 import {
-  CONFLUENCE_MIME_TYPES,
   ConfluenceClientError,
   isConfluenceNotFoundError,
+  MIME_TYPES,
 } from "@dust-tt/types";
 import { Op } from "sequelize";
 import TurndownService from "turndown";
@@ -222,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({
     parents: [makeSpaceInternalId(spaceId)],
     parentId: null,
     title: spaceName,
-    mimeType: CONFLUENCE_MIME_TYPES.SPACE,
+    mimeType: MIME_TYPES.CONFLUENCE.SPACE,
   });
 }
 
@@ -329,7 +329,7 @@ async function upsertConfluencePageToDataSource({
       timestampMs: lastPageVersionCreatedAt.getTime(),
       upsertContext: { sync_type: syncType },
       title: page.title,
-      mimeType: CONFLUENCE_MIME_TYPES.PAGE,
+      mimeType: MIME_TYPES.CONFLUENCE.PAGE,
       async: true,
     });
   }

diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts
@@ -293,7 +293,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
             sourceUrl: repo.url,
             expandable: true,
             permission: "read",
-            dustDocumentId: null,
             lastUpdatedAt: null,
           }))
         );
@@ -364,7 +363,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
               sourceUrl: repo.url + "/issues",
               expandable: false,
               permission: "read",
-              dustDocumentId: null,
               lastUpdatedAt: latestIssue.updatedAt.getTime(),
             });
           }
@@ -378,7 +376,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
               sourceUrl: repo.url + "/discussions",
               expandable: false,
               permission: "read",
-              dustDocumentId: null,
               lastUpdatedAt: latestDiscussion.updatedAt.getTime(),
             });
           }
@@ -392,7 +389,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
               sourceUrl: repo.url,
               expandable: true,
               permission: "read",
-              dustDocumentId: null,
               lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(),
             });
           }
@@ -434,7 +430,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
               sourceUrl: directory.sourceUrl,
               expandable: true,
               permission: "read",
-              dustDocumentId: null,
               lastUpdatedAt: directory.codeUpdatedAt.getTime(),
             });
           });
@@ -448,7 +443,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
               sourceUrl: file.sourceUrl,
               expandable: false,
               permission: "read",
-              dustDocumentId: file.documentId,
               lastUpdatedAt: file.codeUpdatedAt.getTime(),
             });
           });
@@ -610,7 +604,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: repo.url,
         expandable: true,
         permission: "read",
-        dustDocumentId: null,
         lastUpdatedAt: null,
       });
     });
@@ -629,7 +622,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: repo.url + "/issues",
         expandable: false,
         permission: "read",
-        dustDocumentId: null,
         lastUpdatedAt: null,
       });
     });
@@ -646,7 +638,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: repo.url + "/discussions",
         expandable: false,
         permission: "read",
-        dustDocumentId: null,
         lastUpdatedAt: null,
       });
     });
@@ -665,7 +656,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: repo.url + `/issues/${issueNumber}`,
         expandable: false,
         permission: "read",
-        dustDocumentId: getIssueInternalId(repoId, issueNumber),
         lastUpdatedAt: issue.updatedAt.getTime(),
       });
     });
@@ -684,7 +674,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: repo.url + `/discussions/${discussionNumber}`,
         expandable: false,
         permission: "read",
-        dustDocumentId: getDiscussionInternalId(repoId, discussionNumber),
         lastUpdatedAt: discussion.updatedAt.getTime(),
       });
     });
@@ -699,7 +688,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: codeRepo.sourceUrl,
         expandable: true,
         permission: "read",
-        dustDocumentId: null,
         lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(),
       });
     });
@@ -714,7 +702,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: directory.sourceUrl,
         expandable: true,
         permission: "read",
-        dustDocumentId: null,
         lastUpdatedAt: directory.codeUpdatedAt.getTime(),
       });
     });
@@ -729,7 +716,6 @@ export class GithubConnectorManager extends BaseConnectorManager<null> {
         sourceUrl: file.sourceUrl,
         expandable: false,
         permission: "read",
-        dustDocumentId: file.documentId,
         lastUpdatedAt: file.codeUpdatedAt.getTime(),
       });
     });

diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts
@@ -1,5 +1,5 @@
 import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types";
-import { assertNever, GITHUB_MIME_TYPES } from "@dust-tt/types";
+import { assertNever, MIME_TYPES } from "@dust-tt/types";
 import { Context } from "@temporalio/activity";
 import { hash as blake3 } from "blake3";
 import { promises as fs } from "fs";
@@ -308,7 +308,7 @@ export async function githubUpsertIssueActivity(
       sync_type: isBatchSync ? "batch" : "incremental",
     },
     title: issue.title,
-    mimeType: GITHUB_MIME_TYPES.ISSUE,
+    mimeType: MIME_TYPES.GITHUB.ISSUE,
     async: true,
   });
 
@@ -494,7 +494,7 @@ export async function githubUpsertDiscussionActivity(
       sync_type: isBatchSync ? "batch" : "incremental",
     },
     title: discussion.title,
-    mimeType: GITHUB_MIME_TYPES.DISCUSSION,
+    mimeType: MIME_TYPES.GITHUB.DISCUSSION,
     async: true,
   });
 
@@ -960,7 +960,7 @@ export async function githubCodeSyncActivity({
     title: "Code",
     parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: GITHUB_MIME_TYPES.CODE_ROOT,
+    mimeType: MIME_TYPES.GITHUB.CODE_ROOT,
   });
 
   let githubCodeRepository = await GithubCodeRepository.findOne({
@@ -1172,7 +1172,7 @@ export async function githubCodeSyncActivity({
               sync_type: isBatchSync ? "batch" : "incremental",
             },
             title: f.fileName,
-            mimeType: GITHUB_MIME_TYPES.CODE_FILE,
+            mimeType: MIME_TYPES.GITHUB.CODE_FILE,
             async: true,
           });
 
@@ -1216,7 +1216,7 @@ export async function githubCodeSyncActivity({
           parents,
           parentId: parents[1],
           title: d.dirName,
-          mimeType: GITHUB_MIME_TYPES.CODE_DIRECTORY,
+          mimeType: MIME_TYPES.GITHUB.CODE_DIRECTORY,
         });
 
         // Find directory or create it.
@@ -1356,7 +1356,7 @@ export async function githubUpsertRepositoryFolderActivity({
     title: repoName,
     parents: [getRepositoryInternalId(repoId)],
     parentId: null,
-    mimeType: GITHUB_MIME_TYPES.REPOSITORY,
+    mimeType: MIME_TYPES.GITHUB.REPOSITORY,
   });
 }
 
@@ -1377,7 +1377,7 @@ export async function githubUpsertIssuesFolderActivity({
     title: "Issues",
     parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: GITHUB_MIME_TYPES.ISSUES,
+    mimeType: MIME_TYPES.GITHUB.ISSUES,
   });
 }
 
@@ -1401,7 +1401,7 @@ export async function githubUpsertDiscussionsFolderActivity({
       getRepositoryInternalId(repoId),
     ],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: GITHUB_MIME_TYPES.DISCUSSIONS,
+    mimeType: MIME_TYPES.GITHUB.DISCUSSIONS,
   });
 }
 
@@ -1422,6 +1422,6 @@ export async function githubUpsertCodeRootFolderActivity({
     title: "Code",
     parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: GITHUB_MIME_TYPES.CODE_ROOT,
+    mimeType: MIME_TYPES.GITHUB.CODE_ROOT,
   });
 }