upsert documents x2 with both the old and new ID

dust-tt · Jan 2, 2025 · f35462d · f35462d
1 parent 9b681fc
commit f35462d
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 2 deletions.
diff --git a/connectors/src/connectors/zendesk/lib/id_conversions.ts b/connectors/src/connectors/zendesk/lib/id_conversions.ts
@@ -47,6 +47,18 @@ export function getArticleInternalId({
   return `zendesk-article-${connectorId}-${articleId}`;
 }
 
+export function getArticleNewInternalId({
+  connectorId,
+  brandId,
+  articleId,
+}: {
+  connectorId: ModelId;
+  brandId: number;
+  articleId: number;
+}): string {
+  return `zendesk-article-${connectorId}-${brandId}-${articleId}`;
+}
+
 export function getTicketsInternalId({
   connectorId,
   brandId,
@@ -67,6 +79,18 @@ export function getTicketInternalId({
   return `zendesk-ticket-${connectorId}-${ticketId}`;
 }
 
+export function getTicketNewInternalId({
+  connectorId,
+  brandId,
+  ticketId,
+}: {
+  connectorId: ModelId;
+  brandId: number;
+  ticketId: number;
+}): string {
+  return `zendesk-ticket-${connectorId}-${brandId}-${ticketId}`;
+}
+
 /**
  * Conversion from an internalId to an id.
  */

diff --git a/connectors/src/connectors/zendesk/lib/sync_article.ts b/connectors/src/connectors/zendesk/lib/sync_article.ts
@@ -6,7 +6,10 @@ import type {
   ZendeskFetchedSection,
   ZendeskFetchedUser,
 } from "@connectors/@types/node-zendesk";
-import { getArticleInternalId } from "@connectors/connectors/zendesk/lib/id_conversions";
+import {
+  getArticleInternalId,
+  getArticleNewInternalId,
+} from "@connectors/connectors/zendesk/lib/id_conversions";
 import {
   deleteDataSourceDocument,
   renderDocumentTitleAndContent,
@@ -171,6 +174,34 @@ export async function syncArticle({
       mimeType: "application/vnd.dust.zendesk.article",
       async: true,
     });
+
+    // TODO(2025-01-02 aubin): stop upserting documents x2 once the migration of internal IDs is done.
+    const newDocumentId = getArticleNewInternalId({
+      connectorId,
+      brandId: category.brandId,
+      articleId: article.id,
+    });
+
+    await upsertDataSourceDocument({
+      dataSourceConfig,
+      documentId: newDocumentId,
+      documentContent,
+      documentUrl: article.html_url,
+      timestampMs: updatedAt.getTime(),
+      tags: [
+        `title:${article.title}`,
+        `createdAt:${createdAt.getTime()}`,
+        `updatedAt:${updatedAt.getTime()}`,
+      ],
+      parents: [newDocumentId, ...parents.slice(1)],
+      parentId: parents[1],
+      loggerArgs: { ...loggerArgs, articleId: article.id },
+      upsertContext: { sync_type: "batch" },
+      title: article.title,
+      mimeType: "application/vnd.dust.zendesk.article",
+      async: true,
+    });
+
     await articleInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) });
   } else {
     logger.warn(

diff --git a/connectors/src/connectors/zendesk/lib/sync_ticket.ts b/connectors/src/connectors/zendesk/lib/sync_ticket.ts
@@ -6,7 +6,10 @@ import type {
   ZendeskFetchedTicketComment,
   ZendeskFetchedUser,
 } from "@connectors/@types/node-zendesk";
-import { getTicketInternalId } from "@connectors/connectors/zendesk/lib/id_conversions";
+import {
+  getTicketInternalId,
+  getTicketNewInternalId,
+} from "@connectors/connectors/zendesk/lib/id_conversions";
 import {
   deleteDataSourceDocument,
   renderDocumentTitleAndContent,
@@ -230,6 +233,35 @@ ${comments
       mimeType: "application/vnd.dust.zendesk.ticket",
       async: true,
     });
+
+    // TODO(2025-01-02 aubin): stop upserting documents x2 once the migration of internal IDs is done.
+    const newDocumentId = getTicketNewInternalId({
+      connectorId,
+      brandId,
+      ticketId: ticket.id,
+    });
+
+    await upsertDataSourceDocument({
+      dataSourceConfig,
+      documentId: newDocumentId,
+      documentContent,
+      documentUrl: ticket.url,
+      timestampMs: updatedAtDate.getTime(),
+      tags: [
+        ...ticket.tags,
+        `title:${ticket.subject}`,
+        `updatedAt:${updatedAtDate.getTime()}`,
+        `createdAt:${createdAtDate.getTime()}`,
+      ],
+      parents: [newDocumentId, ...parents.slice(1)],
+      parentId: parents[1],
+      loggerArgs: { ...loggerArgs, ticketId: ticket.id },
+      upsertContext: { sync_type: "batch" },
+      title: ticket.subject,
+      mimeType: "application/vnd.dust.zendesk.ticket",
+      async: true,
+    });
+
     await ticketInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) });
   } else {
     logger.warn(