Merge branch 'main' into thomas/extension-region

connectors/src/api/admin.ts

@@ -21,6 +21,10 @@ const whitelistedCommands = [
     command: "find-url",
   },
   { majorCommand: "slack", command: "whitelist-bot" },
+  {
+    majorCommand: "connectors",
+    command: "set-error",
+  },
 ];
 
 const _adminAPIHandler = async (

connectors/src/connectors/confluence/lib/permissions.ts

@@ -34,23 +34,30 @@ function isConfluenceSpaceModel(
   );
 }
 
+export function getConfluenceSpaceUrl(
+  space: ConfluenceSpace | ConfluenceSpaceType,
+  baseUrl: string
+) {
+  const urlSuffix = isConfluenceSpaceModel(space)
+    ? space.urlSuffix
+    : space._links.webui;
+  return `${baseUrl}/wiki${urlSuffix}`;
+}
+
 export function createContentNodeFromSpace(
   space: ConfluenceSpace | ConfluenceSpaceType,
   baseUrl: string,
   permission: ConnectorPermission,
   { isExpandable }: { isExpandable: boolean }
 ): ContentNode {
-  const urlSuffix = isConfluenceSpaceModel(space)
-    ? space.urlSuffix
-    : space._links.webui;
   const spaceId = isConfluenceSpaceModel(space) ? space.spaceId : space.id;
 
   return {
     internalId: makeSpaceInternalId(spaceId),
     parentInternalId: null,
     type: "folder",
     title: space.name || "Unnamed Space",
-    sourceUrl: `${baseUrl}/wiki${urlSuffix}`,
+    sourceUrl: getConfluenceSpaceUrl(space, baseUrl),
     expandable: isExpandable,
     permission,
     lastUpdatedAt: null,

connectors/src/connectors/confluence/temporal/activities.ts

@@ -23,6 +23,7 @@ import {
   makePageInternalId,
   makeSpaceInternalId,
 } from "@connectors/connectors/confluence/lib/internal_ids";
+import { getConfluenceSpaceUrl } from "@connectors/connectors/confluence/lib/permissions";
 import { makeConfluenceDocumentUrl } from "@connectors/connectors/confluence/temporal/workflow_ids";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
@@ -209,20 +210,29 @@ export async function confluenceUpsertSpaceFolderActivity({
   connectorId,
   spaceId,
   spaceName,
+  baseUrl,
 }: {
   connectorId: ModelId;
   spaceId: string;
   spaceName: string;
+  baseUrl: string;
 }) {
   const connector = await fetchConfluenceConnector(connectorId);
 
+  const spaceInDb = await ConfluenceSpace.findOne({
+    attributes: ["urlSuffix"],
+    where: { connectorId, spaceId },
+  });
+
   await upsertDataSourceFolder({
     dataSourceConfig: dataSourceConfigFromConnector(connector),
     folderId: makeSpaceInternalId(spaceId),
     parents: [makeSpaceInternalId(spaceId)],
     parentId: null,
     title: spaceName,
     mimeType: MIME_TYPES.CONFLUENCE.SPACE,
+    sourceUrl:
+      spaceInDb?.urlSuffix && getConfluenceSpaceUrl(spaceInDb, baseUrl),
   });
 }
 

connectors/src/connectors/confluence/temporal/workflows.ts

@@ -148,7 +148,7 @@ export async function confluenceSpaceSyncWorkflow(
   const confluenceConfig = await fetchConfluenceConfigurationActivity(
     params.connectorId
   );
-  const { cloudId: confluenceCloudId } = confluenceConfig;
+  const { cloudId: confluenceCloudId, url: baseUrl } = confluenceConfig;
 
   const spaceName = await confluenceGetSpaceNameActivity({
     ...params,
@@ -163,6 +163,7 @@ export async function confluenceSpaceSyncWorkflow(
     connectorId,
     spaceId,
     spaceName,
+    baseUrl,
   });
 
   const allowedRootPageIds = await fetchAndUpsertRootPagesActivity({

connectors/src/connectors/google_drive/index.ts

@@ -38,6 +38,7 @@ import {
   isGoogleDriveFolder,
   isGoogleDriveSpreadSheetFile,
 } from "@connectors/connectors/google_drive/temporal/mime_types";
+import type { Sheet } from "@connectors/connectors/google_drive/temporal/spreadsheets";
 import {
   driveObjectToDustType,
   getAuthObject,
@@ -67,6 +68,7 @@ import { terminateAllWorkflowsForConnectorId } from "@connectors/lib/temporal";
 import logger from "@connectors/logger/logger";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 import type { DataSourceConfig } from "@connectors/types/data_source_config.js";
+import type { GoogleDriveObjectType } from "@connectors/types/google_drive";
 import { FILE_ATTRIBUTES_TO_FETCH } from "@connectors/types/google_drive";
 
 export class GoogleDriveConnectorManager extends BaseConnectorManager<null> {
@@ -706,7 +708,7 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager<null> {
       type: "database",
       title: s.name || "",
       lastUpdatedAt: s.updatedAt.getTime() || null,
-      sourceUrl: `https://docs.google.com/spreadsheets/d/${s.driveFileId}/edit#gid=${s.driveSheetId}`,
+      sourceUrl: getSourceUrlForGoogleDriveSheet(s),
       expandable: false,
       permission: "read",
     }));
@@ -976,12 +978,25 @@ async function getFoldersAsContentNodes({
   );
 }
 
-function getSourceUrlForGoogleDriveFiles(f: GoogleDriveFiles): string {
+export function getSourceUrlForGoogleDriveFiles(
+  f: GoogleDriveFiles | GoogleDriveObjectType
+): string {
+  const driveFileId = f instanceof GoogleDriveFiles ? f.driveFileId : f.id;
+
   if (isGoogleDriveSpreadSheetFile(f)) {
-    return `https://docs.google.com/spreadsheets/d/${f.driveFileId}/edit`;
+    return `https://docs.google.com/spreadsheets/d/${driveFileId}/edit`;
   } else if (isGoogleDriveFolder(f)) {
-    return `https://drive.google.com/drive/folders/${f.driveFileId}`;
+    return `https://drive.google.com/drive/folders/${driveFileId}`;
   }
 
-  return `https://drive.google.com/file/d/${f.driveFileId}/view`;
+  return `https://drive.google.com/file/d/${driveFileId}/view`;
+}
+
+export function getSourceUrlForGoogleDriveSheet(
+  s: GoogleDriveSheet | Sheet
+): string {
+  const driveFileId =
+    s instanceof GoogleDriveSheet ? s.driveFileId : s.spreadsheet.id;
+  const driveSheetId = s instanceof GoogleDriveSheet ? s.driveSheetId : s.id;
+  return `https://docs.google.com/spreadsheets/d/${driveFileId}/edit#gid=${driveSheetId}`;
 }

connectors/src/connectors/google_drive/temporal/activities.ts

@@ -8,6 +8,7 @@ import StatsD from "hot-shots";
 import PQueue from "p-queue";
 import { Op } from "sequelize";
 
+import { getSourceUrlForGoogleDriveFiles } from "@connectors/connectors/google_drive";
 import {
   GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID,
   GOOGLE_DRIVE_USER_SPACE_VIRTUAL_DRIVE_ID,
@@ -516,6 +517,7 @@ export async function incrementalSync(
           parentId: parents[1] || null,
           title: driveFile.name ?? "",
           mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER,
+          sourceUrl: getSourceUrlForGoogleDriveFiles(driveFile),
         });
 
         await GoogleDriveFiles.upsert({
@@ -861,6 +863,7 @@ export async function markFolderAsVisited(
     parentId: parents[1] || null,
     title: file.name ?? "",
     mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER,
+    sourceUrl: getSourceUrlForGoogleDriveFiles(file),
   });
 
   await GoogleDriveFiles.upsert({

connectors/src/connectors/google_drive/temporal/mime_types.ts

@@ -1,5 +1,3 @@
-import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive";
-
 export const MIME_TYPES_TO_EXPORT: { [key: string]: string } = {
   "application/vnd.google-apps.document": "text/plain",
   "application/vnd.google-apps.presentation": "text/plain",
@@ -48,7 +46,7 @@ export async function getMimeTypesToSync({
   return mimeTypes;
 }
 
-export function isGoogleDriveFolder(file: GoogleDriveFiles) {
+export function isGoogleDriveFolder(file: { mimeType: string }) {
   return file.mimeType === "application/vnd.google-apps.folder";
 }
 

connectors/src/connectors/google_drive/temporal/spreadsheets.ts

@@ -11,6 +11,7 @@ import type { sheets_v4 } from "googleapis";
 import { google } from "googleapis";
 import type { OAuth2Client } from "googleapis-common";
 
+import { getSourceUrlForGoogleDriveSheet } from "@connectors/connectors/google_drive";
 import { getFileParentsMemoized } from "@connectors/connectors/google_drive/lib/hierarchy";
 import { getInternalId } from "@connectors/connectors/google_drive/temporal/utils";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
@@ -30,7 +31,7 @@ import type { GoogleDriveObjectType } from "@connectors/types/google_drive";
 
 const MAXIMUM_NUMBER_OF_GSHEET_ROWS = 50000;
 
-type Sheet = sheets_v4.Schema$ValueRange & {
+export type Sheet = sheets_v4.Schema$ValueRange & {
   id: number;
   spreadsheet: {
     id: string;
@@ -87,6 +88,7 @@ async function upsertGdriveTable(
     useAppForHeaderDetection: true,
     title: `${spreadsheet.title} - ${title}`,
     mimeType: "application/vnd.google-apps.spreadsheet",
+    sourceUrl: getSourceUrlForGoogleDriveSheet(sheet),
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");

connectors/src/connectors/intercom/temporal/sync_help_center.ts

@@ -230,6 +230,7 @@ export async function upsertCollectionWithChildren({
     parents: collectionParents,
     parentId: collectionParents[1],
     mimeType: MIME_TYPES.INTERCOM.COLLECTION,
+    sourceUrl: collection.url || fallbackCollectionUrl,
   });
 
   // Then we call ourself recursively on the children collections

connectors/src/connectors/notion/temporal/activities.ts

@@ -1830,6 +1830,9 @@ export async function renderAndUpsertPageFromCache({
               parentId: parents[1] || null,
               title: parentDb.title ?? "Untitled Notion Database",
               mimeType: MIME_TYPES.NOTION.DATABASE,
+              sourceUrl:
+                parentDb.notionUrl ??
+                `https://www.notion.so/${parentDb.notionDatabaseId.replace(/-/g, "")}`,
             }),
           localLogger
         );
@@ -2551,6 +2554,9 @@ export async function upsertDatabaseStructuredDataFromCache({
         parentId: parentIds[1] || null,
         title: dbModel.title ?? "Untitled Notion Database",
         mimeType: MIME_TYPES.NOTION.DATABASE,
+        sourceUrl:
+          dbModel.notionUrl ??
+          `https://www.notion.so/${dbModel.notionDatabaseId.replace(/-/g, "")}`,
       }),
     localLogger
   );

connectors/src/connectors/slack/lib/channels.ts

@@ -18,6 +18,7 @@ export type SlackChannelType = {
   slackId: string;
   permission: ConnectorPermission;
   agentConfigurationId: string | null;
+  private: boolean;
 };
 
 export async function updateSlackChannelInConnectorsDb({
@@ -60,6 +61,7 @@ export async function updateSlackChannelInConnectorsDb({
     slackId: channel.slackChannelId,
     permission: channel.permission,
     agentConfigurationId: channel.agentConfigurationId,
+    private: channel.private,
   };
 }
 

connectors/src/connectors/slack/temporal/activities.ts

@@ -257,6 +257,7 @@ export async function syncChannel(
       parents: [slackChannelInternalIdFromSlackChannelId(channelId)],
       mimeType: MIME_TYPES.SLACK.CHANNEL,
       sourceUrl: getSlackChannelSourceUrl(channelId, slackConfiguration),
+      providerVisibility: channel.private ? "private" : "public",
     });
   }
 

connectors/src/connectors/webcrawler/temporal/activities.ts

@@ -293,6 +293,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
             parentId: parents[1] || null,
             title: folder,
             mimeType: MIME_TYPES.WEBCRAWLER.FOLDER,
+            sourceUrl: webCrawlerFolder.url,
           });
 
           createdFolders.add(folder);

connectors/src/lib/data_sources.ts

@@ -9,6 +9,7 @@ import type {
   CoreAPIFolder,
   CoreAPITable,
   PostDataSourceDocumentRequestBody,
+  ProviderVisibility,
 } from "@dust-tt/types";
 import {
   isValidDate,
@@ -1244,6 +1245,7 @@ export async function _upsertDataSourceFolder({
   title,
   mimeType,
   sourceUrl,
+  providerVisibility,
 }: {
   dataSourceConfig: DataSourceConfig;
   folderId: string;
@@ -1253,6 +1255,7 @@ export async function _upsertDataSourceFolder({
   title: string;
   mimeType: string;
   sourceUrl?: string;
+  providerVisibility?: ProviderVisibility;
 }) {
   const now = new Date();
 
@@ -1265,6 +1268,7 @@ export async function _upsertDataSourceFolder({
     parents,
     mimeType,
     sourceUrl: sourceUrl ?? null,
+    providerVisibility: providerVisibility || null,
   });
 
   if (r.isErr()) {

connectors/src/types/google_drive.ts

@@ -17,6 +17,7 @@ export type GoogleDriveObjectType = {
   driveId: string;
   isInSharedDrive: boolean;
 };
+
 export type GoogleDriveFolderType = {
   id: string;
   name: string;

core/src/lib.rs

@@ -56,6 +56,7 @@ pub mod providers {
     pub mod anthropic;
     pub mod deepseek;
     pub mod google_ai_studio;
+    pub mod openai_compatible_helpers;
     pub mod togetherai;
 }
 pub mod http {