Move to ConnectorType.configuration (#4557)

* Tmp commit

Fix managed/update

renderConnectorType()

Fully working for websites

Clean up

Cleaning up the createConnector Interface

Clean up typing in connectors

Removes unknown from connectorsAPI.updateConfiguration

Removing unknown

Cleaning up following @spolu's comments.

Remvoe console.log()

Just removing a comment

Make connectionId non optional

Make sure only webcrawler configuration can be updated by a user

* Fix typing

connectors/src/api/configuration.ts

@@ -0,0 +1,111 @@
+import type {
+  ConnectorType,
+  Result,
+  UpdateConnectorConfigurationType,
+  WithConnectorsAPIErrorReponse,
+} from "@dust-tt/types";
+import {
+  assertNever,
+  ioTsParsePayload,
+  WebCrawlerConfigurationTypeSchema,
+} from "@dust-tt/types";
+import type { Request, Response } from "express";
+
+import { SET_CONNECTOR_CONFIGURATION_BY_TYPE } from "@connectors/connectors";
+import { renderConnectorType } from "@connectors/lib/renderers/connector";
+import { apiError, withLogging } from "@connectors/logger/withlogging";
+import { ConnectorResource } from "@connectors/resources/connector_resource";
+
+type PatchConnectorConfigurationResBody =
+  WithConnectorsAPIErrorReponse<ConnectorType>;
+
+const _patchConnectorConfiguration = async (
+  req: Request<
+    { connector_id: string },
+    PatchConnectorConfigurationResBody,
+    UpdateConnectorConfigurationType
+  >,
+  res: Response<PatchConnectorConfigurationResBody>
+) => {
+  const connector = await ConnectorResource.fetchById(req.params.connector_id);
+  if (!connector) {
+    return apiError(req, res, {
+      api_error: {
+        type: "connector_not_found",
+        message: "Connector not found",
+      },
+      status_code: 404,
+    });
+  }
+
+  let patchRes: Result<void, Error> | null = null;
+  switch (connector.type) {
+    case "webcrawler": {
+      const parseRes = ioTsParsePayload(
+        req.body.configuration,
+        WebCrawlerConfigurationTypeSchema
+      );
+      if (parseRes.isErr()) {
+        return apiError(req, res, {
+          api_error: {
+            type: "invalid_request_error",
+            message: `Invalid configuration: ${parseRes.error.join(", ")}`,
+          },
+          status_code: 400,
+        });
+      }
+      const setConfiguration =
+        SET_CONNECTOR_CONFIGURATION_BY_TYPE[connector.type];
+      patchRes = await setConfiguration(connector.id, parseRes.value);
+      break;
+    }
+
+    case "notion":
+    case "confluence":
+    case "github":
+    case "google_drive":
+    case "intercom":
+    case "slack": {
+      throw new Error(
+        `Connector type ${connector.type} does not support configuration patching`
+      );
+    }
+
+    default: {
+      assertNever(connector.type);
+    }
+  }
+
+  if (patchRes.isErr()) {
+    return apiError(
+      req,
+      res,
+      {
+        api_error: {
+          type: "internal_server_error",
+          message: patchRes.error.message,
+        },
+        status_code: 500,
+      },
+      patchRes.error
+    );
+  }
+
+  const updatedConnector = await ConnectorResource.fetchById(
+    req.params.connector_id
+  );
+  if (!updatedConnector) {
+    return apiError(req, res, {
+      api_error: {
+        type: "connector_not_found",
+        message: "Connector not found",
+      },
+      status_code: 404,
+    });
+  }
+  return res.status(200).json(await renderConnectorType(updatedConnector));
+};
+
+export const patchConnectorConfigurationAPIHandler = withLogging(
+  _patchConnectorConfiguration
+);

connectors/src/api/create_connector.ts

@@ -1,27 +1,22 @@
 import type {
   ConnectorType,
-  CreateConnectorOAuthRequestBodySchema,
-  CreateConnectorUrlRequestBodySchema,
   Result,
   WithConnectorsAPIErrorReponse,
 } from "@dust-tt/types";
 import {
   assertNever,
   ConnectorCreateRequestBodySchema,
+  ioTsParsePayload,
   isConnectorProvider,
-  provider2createConnectorType,
+  WebCrawlerConfigurationTypeSchema,
 } from "@dust-tt/types";
 import type { Request, Response } from "express";
 import { isLeft } from "fp-ts/lib/Either";
-import type * as t from "io-ts";
 import * as reporter from "io-ts-reporters";
 
 import { CREATE_CONNECTOR_BY_TYPE } from "@connectors/connectors";
-import type {
-  ConnectorCreatorOAuth,
-  ConnectorCreatorUrl,
-} from "@connectors/connectors/interface";
 import { errorFromAny } from "@connectors/lib/error";
+import { renderConnectorType } from "@connectors/lib/renderers/connector";
 import logger from "@connectors/logger/logger";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
@@ -56,53 +51,66 @@ const _createConnectorAPIHandler = async (
       });
     }
 
-    const { workspaceAPIKey, dataSourceName, workspaceId, connectorParams } =
-      bodyValidation.right;
-
-    let connectorRes: Result<string, Error>;
-    const createConnectorType =
-      provider2createConnectorType[req.params.connector_provider];
-    switch (createConnectorType) {
-      case "oauth": {
-        const connectorCreator = CREATE_CONNECTOR_BY_TYPE[
-          req.params.connector_provider
-        ] as ConnectorCreatorOAuth;
+    const {
+      workspaceAPIKey,
+      dataSourceName,
+      workspaceId,
+      connectionId,
+      configuration,
+    } = bodyValidation.right;
 
-        const params = connectorParams as t.TypeOf<
-          typeof CreateConnectorOAuthRequestBodySchema
-        >;
+    let connectorRes: Result<string, Error> | null = null;
+    null;
+    switch (req.params.connector_provider) {
+      case "webcrawler": {
+        const connectorCreator =
+          CREATE_CONNECTOR_BY_TYPE[req.params.connector_provider];
+        const configurationRes = ioTsParsePayload(
+          configuration,
+          WebCrawlerConfigurationTypeSchema
+        );
+        if (configurationRes.isErr()) {
+          return apiError(req, res, {
+            status_code: 400,
+            api_error: {
+              type: "invalid_request_error",
+              message: `Invalid request body: ${configurationRes.error}`,
+            },
+          });
+        }
         connectorRes = await connectorCreator(
           {
             workspaceAPIKey: workspaceAPIKey,
             dataSourceName: dataSourceName,
             workspaceId: workspaceId,
           },
-          params.connectionId
+          connectionId,
+          configurationRes.value
         );
         break;
       }
-      case "url": {
-        const connectorCreator = CREATE_CONNECTOR_BY_TYPE[
-          req.params.connector_provider
-        ] as ConnectorCreatorUrl;
 
-        const params = connectorParams as t.TypeOf<
-          typeof CreateConnectorUrlRequestBodySchema
-        >;
+      case "github":
+      case "notion":
+      case "confluence":
+      case "google_drive":
+      case "intercom":
+      case "slack": {
+        const connectorCreator =
+          CREATE_CONNECTOR_BY_TYPE[req.params.connector_provider];
         connectorRes = await connectorCreator(
           {
             workspaceAPIKey: workspaceAPIKey,
             dataSourceName: dataSourceName,
             workspaceId: workspaceId,
           },
-          params
+          connectionId,
+          null
         );
         break;
       }
-
-      default: {
-        assertNever(createConnectorType);
-      }
+      default:
+        assertNever(req.params.connector_provider);
     }
 
     if (connectorRes.isErr()) {
@@ -116,6 +124,7 @@ const _createConnectorAPIHandler = async (
     }
 
     const connector = await ConnectorResource.fetchById(connectorRes.value);
+
     if (!connector) {
       return apiError(req, res, {
         status_code: 500,
@@ -126,17 +135,7 @@ const _createConnectorAPIHandler = async (
       });
     }
 
-    return res.status(200).json({
-      id: connector.id.toString(),
-      type: connector.type,
-      workspaceId: connector.workspaceId,
-      dataSourceName: connector.dataSourceName,
-      lastSyncStatus: connector.lastSyncStatus,
-      lastSyncStartTime: connector.lastSyncStartTime?.getTime(),
-      lastSyncSuccessfulTime: connector.lastSyncSuccessfulTime?.getTime(),
-      firstSuccessfulSyncTime: connector.firstSuccessfulSyncTime?.getTime(),
-      firstSyncProgress: connector.firstSyncProgress,
-    });
+    return res.status(200).json(await renderConnectorType(connector));
   } catch (e) {
     logger.error(errorFromAny(e), "Error in createConnectorAPIHandler");
     return apiError(req, res, {

connectors/src/api/get_connector.ts

@@ -6,6 +6,7 @@ import type { Request, Response } from "express";
 
 import { GithubDiscussion, GithubIssue } from "@connectors/lib/models/github";
 import { NotionPage } from "@connectors/lib/models/notion";
+import { renderConnectorType } from "@connectors/lib/renderers/connector";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 
@@ -68,19 +69,7 @@ const _getConnector = async (
     }
   }
 
-  return res.status(200).json({
-    id: connector.id.toString(),
-    type: connector.type,
-    workspaceId: connector.workspaceId,
-    dataSourceName: connector.dataSourceName,
-    lastSyncStatus: connector.lastSyncStatus,
-    lastSyncStartTime: connector.lastSyncStartTime?.getTime(),
-    lastSyncFinishTime: connector.lastSyncFinishTime?.getTime(),
-    lastSyncSuccessfulTime: connector.lastSyncSuccessfulTime?.getTime(),
-    firstSuccessfulSyncTime: connector.firstSuccessfulSyncTime?.getTime(),
-    firstSyncProgress,
-    errorType: connector.errorType || undefined,
-  });
+  return res.status(200).json(await renderConnectorType(connector));
 };
 
 export const getConnectorAPIHandler = withLogging(_getConnector);

connectors/src/api/update_connector.ts

@@ -1,21 +1,10 @@
-import type {
-  CreateConnectorOAuthRequestBody,
-  CreateConnectorUrlRequestBody,
-  WithConnectorsAPIErrorReponse,
-} from "@dust-tt/types";
-import {
-  provider2createConnectorType,
-  UpdateConnectorRequestBodySchema,
-} from "@dust-tt/types";
+import type { WithConnectorsAPIErrorReponse } from "@dust-tt/types";
+import { UpdateConnectorRequestBodySchema } from "@dust-tt/types";
 import type { Request, Response } from "express";
 import { isLeft } from "fp-ts/lib/Either";
 import * as reporter from "io-ts-reporters";
 
 import { UPDATE_CONNECTOR_BY_TYPE } from "@connectors/connectors";
-import type {
-  ConnectorUpdaterOAuth,
-  ConnectorUpdaterUrl,
-} from "@connectors/connectors/interface";
 import { apiError, withLogging } from "@connectors/logger/withlogging";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 
@@ -26,7 +15,7 @@ type ConnectorUpdateResBody = WithConnectorsAPIErrorReponse<{
   connectorId: string;
 }>;
 
-const _getConnectorUpdateAPIHandler = async (
+const _postConnectorUpdateAPIHandler = async (
   req: Request<{ connector_id: string }, ConnectorUpdateReqBody>,
   res: Response<ConnectorUpdateResBody>
 ) => {
@@ -53,7 +42,7 @@ const _getConnectorUpdateAPIHandler = async (
     });
   }
 
-  const { connectorParams } = bodyValidation.right;
+  const { connectionId } = bodyValidation.right;
 
   const connector = await ConnectorResource.fetchById(req.params.connector_id);
   if (!connector) {
@@ -66,27 +55,10 @@ const _getConnectorUpdateAPIHandler = async (
     });
   }
 
-  const updateRes = await (async () => {
-    switch (provider2createConnectorType[connector.type]) {
-      case "oauth": {
-        const connectorUpdater = UPDATE_CONNECTOR_BY_TYPE[
-          connector.type
-        ] as ConnectorUpdaterOAuth;
-        const params = connectorParams as CreateConnectorOAuthRequestBody;
-
-        return connectorUpdater(connector.id, {
-          connectionId: params.connectionId,
-        });
-      }
-      case "url": {
-        const params = connectorParams as CreateConnectorUrlRequestBody;
-        const connectorUpdater = UPDATE_CONNECTOR_BY_TYPE[
-          connector.type
-        ] as ConnectorUpdaterUrl;
-        return connectorUpdater(connector.id, params);
-      }
-    }
-  })();
+  const connectorUpdater = UPDATE_CONNECTOR_BY_TYPE[connector.type];
+  const updateRes = await connectorUpdater(connector.id, {
+    connectionId: connectionId,
+  });
 
   if (updateRes.isErr()) {
     if (updateRes.error.type === "connector_oauth_target_mismatch") {
@@ -110,6 +82,6 @@ const _getConnectorUpdateAPIHandler = async (
   });
 };
 
-export const getConnectorUpdateAPIHandler = withLogging(
-  _getConnectorUpdateAPIHandler
+export const postConnectorUpdateAPIHandler = withLogging(
+  _postConnectorUpdateAPIHandler
 );