From 87d36cf00b3a7ffb123662b4a58d11201ce67e22 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 18:10:38 +0100 Subject: [PATCH] feat: add link to maintained document in tracker email --- front/lib/api/tracker.ts | 99 ++++++++++++++++++------- front/lib/models/doc_tracker.ts | 8 +- front/lib/resources/tracker_resource.ts | 16 ++++ types/src/front/tracker.ts | 16 ++-- 4 files changed, 103 insertions(+), 36 deletions(-) diff --git a/front/lib/api/tracker.ts b/front/lib/api/tracker.ts index 38627bf4f6c67..0793d6854eb74 100644 --- a/front/lib/api/tracker.ts +++ b/front/lib/api/tracker.ts @@ -1,5 +1,5 @@ import type { TrackerGenerationToProcess } from "@dust-tt/types"; -import { concurrentExecutor, CoreAPI } from "@dust-tt/types"; +import { concurrentExecutor, CoreAPI, removeNulls } from "@dust-tt/types"; import _ from "lodash"; import config from "@app/lib/api/config"; @@ -127,15 +127,32 @@ const _sendTrackerWithGenerationEmail = async ({ localLogger: Logger; }): Promise => { const coreAPI = new CoreAPI(config.getCoreAPIConfig(), localLogger); - const generationsByDataSources = _.groupBy(generations, "dataSource.id"); - const documentsById = new Map(); + const dataSourceById = _.keyBy( + removeNulls( + generations.map((g) => [g.dataSource, g.maintainedDataSource]).flat() + ), + "id" + ); + const docsToFetchByDataSourceId = _.mapValues( + _.groupBy( + generations.map((g) => ({ + dataSourceId: g.dataSource.id, + documentIds: removeNulls([g.documentId, g.maintainedDocumentId]), + })), + "dataSourceId" + ), + (docs) => docs.map((d) => d.documentIds).flat() + ); + const documentsByIdentifier = new Map< + string, + { name: string; url: string | null } + >(); // Fetch documents for each data source in parallel. await concurrentExecutor( - Object.entries(generationsByDataSources), - async ([, generations]) => { - const dataSource = generations[0].dataSource; - const documentIds = [...new Set(generations.map((g) => g.documentId))]; + Object.entries(docsToFetchByDataSourceId), + async ([dataSourceId, documentIds]) => { + const dataSource = dataSourceById[dataSourceId]; const docsResult = await coreAPI.getDataSourceDocuments({ projectId: dataSource.dustAPIProjectId, @@ -156,7 +173,7 @@ const _sendTrackerWithGenerationEmail = async ({ } docsResult.value.documents.forEach((doc) => { - documentsById.set(doc.document_id, { + documentsByIdentifier.set(`${dataSource.id}__${doc.document_id}`, { name: doc.title ?? "Unknown document", url: doc.source_url ?? null, }); @@ -165,31 +182,59 @@ const _sendTrackerWithGenerationEmail = async ({ { concurrency: 5 } ); - const generationBody = generations.map((generation) => { - const doc = documentsById.get(generation.documentId) ?? { - name: "Unknown document", - url: null, - }; - - const title = doc.url - ? `${doc.name}` - : `[${doc.name}]`; - - return [ - `Changes in document ${title} from ${generation.dataSource.name}:`, - generation.thinking && ``, - `

${generation.content}.

`, - ] - .filter(Boolean) - .join(""); - }); + const generationBody = await Promise.all( + generations.map((g) => { + // const getDoc = (dataSourceId: number, documentId: string) => + // documentsByIdentifier.get(`${dataSourceId}__${documentId}`) ?? { + // name: "Unknown document", + // url: null, + // }; + + const doc = documentsByIdentifier.get( + `${g.dataSource.id}__${g.documentId}` + ) ?? { + name: "Unknown document", + url: null, + }; + const maintainedDoc = g.maintainedDataSource + ? documentsByIdentifier.get( + `${g.maintainedDataSource.id}__${g.maintainedDocumentId}` + ) ?? null + : null; + + const title = doc.url + ? `${doc.name}` + : `[${doc.name}]`; + + let maintainedTitle: string | null = null; + if (maintainedDoc) { + maintainedTitle = maintainedDoc.url + ? `${maintainedDoc.name}` + : `[${maintainedDoc.name}]`; + } + + let body = `Changes in document ${title} from ${g.dataSource.name}`; + if (maintainedTitle && g.maintainedDataSource) { + body += ` might affect ${maintainedTitle} from ${g.maintainedDataSource.name}`; + } + body += `:`; + + // TODO(@fontanierh): maybe add thinking back in. + // if (g.thinking) { + // body += ``; + // } + + body += `

${g.content.replace(/\n/g, "
")}.

`; + return body; + }) + ); const body = `

We have new suggestions for your tracker ${name}:

${generations.length} recommendations were generated due to changes in watched documents.



-${generationBody.join("
")} +${generationBody.join("
")} `; await sendEmailWithTemplate({ diff --git a/front/lib/models/doc_tracker.ts b/front/lib/models/doc_tracker.ts index 3b9af1c38b214..5e40b00c02657 100644 --- a/front/lib/models/doc_tracker.ts +++ b/front/lib/models/doc_tracker.ts @@ -237,14 +237,16 @@ export class TrackerGenerationModel extends SoftDeletableModel; declare dataSourceId: ForeignKey; declare documentId: string; - declare maintainedDocumentDataSourceId: ForeignKey; - declare maintainedDocumentId: string; + declare maintainedDocumentDataSourceId: ForeignKey< + DataSourceModel["id"] + > | null; + declare maintainedDocumentId: string | null; declare consumedAt: Date | null; declare trackerConfiguration: NonAttribute; declare dataSource: NonAttribute; - declare maintainedDocumentDataSource: NonAttribute; + declare maintainedDocumentDataSource: NonAttribute | null; } TrackerGenerationModel.init( diff --git a/front/lib/resources/tracker_resource.ts b/front/lib/resources/tracker_resource.ts index 7910e389db465..3a197b57c6c9f 100644 --- a/front/lib/resources/tracker_resource.ts +++ b/front/lib/resources/tracker_resource.ts @@ -437,6 +437,11 @@ export class TrackerConfigurationResource extends ResourceWithSpace