From 98e75d8e26d09849c0d81c6733ac7b49734e3b89 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 13 Jan 2025 14:03:41 +0100 Subject: [PATCH 1/2] Removed usage of get_content_node_parents --- .../src/api/get_content_node_parents.ts | 77 ------------------- connectors/src/api_server.ts | 6 -- front/temporal/tracker/activities.ts | 10 ++- types/src/front/lib/connectors_api.ts | 35 +-------- 4 files changed, 10 insertions(+), 118 deletions(-) delete mode 100644 connectors/src/api/get_content_node_parents.ts diff --git a/connectors/src/api/get_content_node_parents.ts b/connectors/src/api/get_content_node_parents.ts deleted file mode 100644 index 26a12ebc4062..000000000000 --- a/connectors/src/api/get_content_node_parents.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { WithConnectorsAPIErrorReponse } from "@dust-tt/types"; -import type { Request, Response } from "express"; -import { isLeft } from "fp-ts/lib/Either"; -import * as t from "io-ts"; -import * as reporter from "io-ts-reporters"; - -import type { ContentNodeParentIdsBlob } from "@connectors/lib/api/content_nodes"; -import { getParentIdsForContentNodes } from "@connectors/lib/api/content_nodes"; -import logger from "@connectors/logger/logger"; -import { apiError, withLogging } from "@connectors/logger/withlogging"; -import { ConnectorResource } from "@connectors/resources/connector_resource"; - -const GetContentNodesParentsRequestBodySchema = t.type({ - internalIds: t.array(t.string), -}); - -export type GetContentNodesParentsRequestBody = t.TypeOf< - typeof GetContentNodesParentsRequestBodySchema ->; - -type GetContentNodesResponseBody = WithConnectorsAPIErrorReponse<{ - nodes: ContentNodeParentIdsBlob[]; -}>; - -const _getContentNodesParents = async ( - req: Request< - { connector_id: string }, - GetContentNodesResponseBody, - GetContentNodesParentsRequestBody - >, - res: Response -) => { - const connector = await ConnectorResource.fetchById(req.params.connector_id); - if (!connector) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "connector_not_found", - message: "Connector not found", - }, - }); - } - - const bodyValidation = GetContentNodesParentsRequestBodySchema.decode( - req.body - ); - if (isLeft(bodyValidation)) { - const pathError = reporter.formatValidationErrors(bodyValidation.left); - return apiError(req, res, { - status_code: 400, - api_error: { - type: "invalid_request_error", - message: `Invalid request body: ${pathError}`, - }, - }); - } - - const { internalIds } = bodyValidation.right; - - const parentsRes = await getParentIdsForContentNodes(connector, internalIds); - if (parentsRes.isErr()) { - logger.error(parentsRes.error, "Failed to get content node parents."); - return apiError(req, res, { - status_code: 500, - api_error: { - type: "internal_server_error", - message: parentsRes.error.message, - }, - }); - } - - return res.status(200).json({ nodes: parentsRes.value }); -}; - -export const getContentNodesParentsAPIHandler = withLogging( - _getContentNodesParents -); diff --git a/connectors/src/api_server.ts b/connectors/src/api_server.ts index 11490c655e99..ff0b3ebb9efd 100644 --- a/connectors/src/api_server.ts +++ b/connectors/src/api_server.ts @@ -13,7 +13,6 @@ import { getConnectorsAPIHandler, } from "@connectors/api/get_connector"; import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions"; -import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents"; import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes"; import { pauseConnectorAPIHandler } from "@connectors/api/pause_connector"; import { resumeConnectorAPIHandler } from "@connectors/api/resume_connector"; @@ -112,11 +111,6 @@ export function startServer(port: number) { "/connectors/:connector_id/permissions", getConnectorPermissionsAPIHandler ); - app.post( - // must be POST because of body - "/connectors/:connector_id/content_nodes/parents", - getContentNodesParentsAPIHandler - ); app.post( // must be POST because of body "/connectors/:connector_id/content_nodes", diff --git a/front/temporal/tracker/activities.ts b/front/temporal/tracker/activities.ts index 35749331c931..d8e4f5b40982 100644 --- a/front/temporal/tracker/activities.ts +++ b/front/temporal/tracker/activities.ts @@ -11,6 +11,7 @@ import { Err, GPT_4O_MODEL_CONFIG, Ok, + removeNulls, } from "@dust-tt/types"; import { Context } from "@temporalio/activity"; import _ from "lodash"; @@ -484,18 +485,19 @@ async function getTrackersToRun( config.getConnectorsAPIConfig(), logger ); - const parentsResult = await connectorsAPI.getContentNodesParents({ + const parentsResult = await connectorsAPI.getContentNodes({ connectorId: dataSource.connectorId, internalIds: [documentId], + includeParents: true, }); if (parentsResult.isErr()) { throw parentsResult.error; } - docParentIds = [ + docParentIds = removeNulls([ documentId, - ...parentsResult.value.nodes.flatMap((node) => node.parents), - ]; + ...parentsResult.value.nodes.flatMap((node) => node.parentInternalIds), + ]); } return TrackerConfigurationResource.fetchAllWatchedForDocument(auth, { diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index f31ee8d98ce7..91b94f5e50a3 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -1,4 +1,7 @@ -import { AdminCommandType, AdminResponseType } from "../../connectors/admin/cli"; +import { + AdminCommandType, + AdminResponseType, +} from "../../connectors/admin/cli"; import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api"; import { UpdateConnectorConfigurationType } from "../../connectors/api_handlers/connector_configuration"; import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector"; @@ -495,36 +498,6 @@ export class ConnectorsAPI { return this._resultFromResponse(res); } - async getContentNodesParents({ - connectorId, - internalIds, - }: { - connectorId: string; - internalIds: string[]; - }): Promise< - ConnectorsAPIResponse<{ - nodes: { - internalId: string; - parents: string[]; - }[]; - }> - > { - const res = await this._fetchWithError( - `${this._url}/connectors/${encodeURIComponent( - connectorId - )}/content_nodes/parents`, - { - method: "POST", - headers: this.getDefaultHeaders(), - body: JSON.stringify({ - internalIds, - }), - } - ); - - return this._resultFromResponse(res); - } - async getContentNodes({ connectorId, includeParents, From 29b231583b0ca1b1343a76d4d9bcb68c0bafdc69 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 13 Jan 2025 14:10:09 +0100 Subject: [PATCH 2/2] front linter --- types/src/front/data_source_view.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/types/src/front/data_source_view.ts b/types/src/front/data_source_view.ts index cf0f01c0cd44..82fc37c116cf 100644 --- a/types/src/front/data_source_view.ts +++ b/types/src/front/data_source_view.ts @@ -1,6 +1,11 @@ import { ModelId } from "../shared/model_id"; import { DataSourceViewCategory } from "./api_handlers/public/spaces"; -import { ConnectorStatusDetails, DataSourceType, DataSourceWithAgentsUsageType, EditedByUser } from "./data_source"; +import { + ConnectorStatusDetails, + DataSourceType, + DataSourceWithAgentsUsageType, + EditedByUser, +} from "./data_source"; import { ContentNode } from "./lib/connectors_api"; export interface DataSourceViewType {