[connectors] Move all Zendesk garbage collection to the gc workflow (#…

…8748)

* feat: move the brand garbage collection from the sync workflow to the gc workflow

* refactor: add a function that deletes a brand for consistency

* refactor: rename getZendeskCategoriesActivity with fetch (gonna use the name with a get)

* refactor: move the deletion of the category to the function deleteCategory

* feat: add a gc for the empty categories

* lint

* feat: only revoke permissions in the setPermissions instead of deleting the brand

* fix: fix fetchBatchByBrandId

* refactor: remove an unused function

* feat: batch the ticket delete function

* feat: batch all activities relative to deleting huge amounts of tickets or articles

* refactor: rename garbageCollectCategories into removeEmptyCategories

* refactor: rename garbageCollectTicket/Article
 into more explicit names

* 📝

* feat: add a cleanup for the categories

* refactor: shorten a few function names

* refactor: remove an unused activity

* bump the workflow version

* fix: avoid unbounded promises by adding concurrent execution

connectors/src/connectors/zendesk/lib/data_cleanup.ts

@@ -0,0 +1,43 @@
+import { getArticleInternalId } from "@connectors/connectors/zendesk/lib/id_conversions";
+import { concurrentExecutor } from "@connectors/lib/async_utils";
+import { deleteFromDataSource } from "@connectors/lib/data_sources";
+import {
+  ZendeskArticleResource,
+  ZendeskCategoryResource,
+} from "@connectors/resources/zendesk_resources";
+import type { DataSourceConfig } from "@connectors/types/data_source_config";
+
+/**
+ * Deletes all the data stored in the db and in the data source relative to a category (articles).
+ */
+export async function deleteCategory({
+  connectorId,
+  categoryId,
+  dataSourceConfig,
+}: {
+  connectorId: number;
+  categoryId: number;
+  dataSourceConfig: DataSourceConfig;
+}) {
+  /// deleting the articles in the data source
+  const articles = await ZendeskArticleResource.fetchByCategoryId({
+    connectorId,
+    categoryId,
+  });
+  await concurrentExecutor(
+    articles,
+    (article) =>
+      deleteFromDataSource(
+        dataSourceConfig,
+        getArticleInternalId(connectorId, article.articleId)
+      ),
+    { concurrency: 10 }
+  );
+  /// deleting the articles stored in the db
+  await ZendeskArticleResource.deleteByCategoryId({
+    connectorId,
+    categoryId,
+  });
+  // deleting the category stored in the db
+  await ZendeskCategoryResource.deleteByCategoryId({ connectorId, categoryId });
+}

connectors/src/connectors/zendesk/temporal/activities.ts

@@ -1,9 +1,6 @@
 import type { ModelId } from "@dust-tt/types";
 
-import {
-  getArticleInternalId,
-  getTicketInternalId,
-} from "@connectors/connectors/zendesk/lib/id_conversions";
+import { deleteCategory } from "@connectors/connectors/zendesk/lib/data_cleanup";
 import { syncArticle } from "@connectors/connectors/zendesk/lib/sync_article";
 import {
   deleteTicket,
@@ -21,19 +18,15 @@ import {
 import { ZENDESK_BATCH_SIZE } from "@connectors/connectors/zendesk/temporal/config";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
 import { concurrentExecutor } from "@connectors/lib/async_utils";
-import { deleteFromDataSource } from "@connectors/lib/data_sources";
 import { ZendeskTimestampCursors } from "@connectors/lib/models/zendesk";
 import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status";
 import logger from "@connectors/logger/logger";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 import {
-  ZendeskArticleResource,
   ZendeskBrandResource,
   ZendeskCategoryResource,
   ZendeskConfigurationResource,
-  ZendeskTicketResource,
 } from "@connectors/resources/zendesk_resources";
-import type { DataSourceConfig } from "@connectors/types/data_source_config";
 
 /**
  * This activity is responsible for updating the lastSyncStartTime of the connector to now.
@@ -80,14 +73,10 @@ export async function saveZendeskConnectorSuccessSync(
 
 /**
  * This activity is responsible for syncing a Brand.
- * It does not sync the content inside the Brand, only the Brand data in itself.
- *
- * It is going to update the name of the Brand if it has changed.
- * If the Brand is not allowed anymore, it will delete all its data.
- * If the Brand is not present on Zendesk anymore, it will delete all its data as well.
- * If the Help Center has no readable category anymore, we delete the Help Center data.
+ * It does not sync the content inside the Brand, only the Brand data in itself (name, url, subdomain, lastUpsertedTs).
+ * If the brand is not found in Zendesk, it deletes it.
  *
- * @returns the updated permissions of the Brand.
+ * @returns the permissions of the Brand.
  */
 export async function syncZendeskBrandActivity({
   connectorId,
@@ -102,7 +91,6 @@ export async function syncZendeskBrandActivity({
   if (!connector) {
     throw new Error("[Zendesk] Connector not found.");
   }
-  const dataSourceConfig = dataSourceConfigFromConnector(connector);
 
   const brandInDb = await ZendeskBrandResource.fetchByBrandId({
     connectorId,
@@ -114,62 +102,28 @@ export async function syncZendeskBrandActivity({
     );
   }
 
-  // deleting the tickets/help center if not allowed anymore
-  if (brandInDb.ticketsPermission === "none") {
-    await deleteBrandTickets({ connectorId, brandId, dataSourceConfig });
-  }
-  if (brandInDb.helpCenterPermission === "none") {
-    await deleteBrandHelpCenter({ connectorId, brandId, dataSourceConfig });
-  }
-
-  // if all rights were revoked, we delete the brand data.
-  if (
-    brandInDb.helpCenterPermission === "none" &&
-    brandInDb.ticketsPermission === "none"
-  ) {
-    await brandInDb.delete();
-    return { helpCenterAllowed: false, ticketsAllowed: false };
-  }
-
-  // if the brand is not on Zendesk anymore, we delete it
   const zendeskApiClient = createZendeskClient(
     await getZendeskSubdomainAndAccessToken(connector.connectionId)
   );
   const {
     result: { brand: fetchedBrand },
   } = await zendeskApiClient.brand.show(brandId);
-  if (!fetchedBrand) {
-    await Promise.all([
-      deleteBrandHelpCenter({ connectorId, brandId, dataSourceConfig }),
-      deleteBrandTickets({ connectorId, brandId, dataSourceConfig }),
-    ]);
-    await brandInDb.delete();
-    return { helpCenterAllowed: false, ticketsAllowed: false };
-  }
 
-  // if there are no read permissions on any category, we delete the help center
-  const categoriesWithReadPermissions =
-    await ZendeskCategoryResource.fetchByBrandIdReadOnly({
-      connectorId,
-      brandId,
-    });
-  const noMoreAllowedCategories = categoriesWithReadPermissions.length === 0;
-
-  if (noMoreAllowedCategories) {
-    await deleteBrandHelpCenter({ connectorId, brandId, dataSourceConfig });
-    // if the tickets and all children categories are not allowed anymore, we delete the brand data
-    if (brandInDb.ticketsPermission !== "read") {
-      await brandInDb.delete();
-      return { helpCenterAllowed: false, ticketsAllowed: false };
-    }
+  // if the brand is not on Zendesk anymore, we delete it
+  if (!fetchedBrand) {
+    await brandInDb.revokeTicketsPermissions();
     await brandInDb.revokeHelpCenterPermissions();
+    return { helpCenterAllowed: false, ticketsAllowed: false };
   }
 
-  // otherwise, we update the brand name and lastUpsertedTs
+  // otherwise, we update the brand data and lastUpsertedTs
   await brandInDb.update({
     name: fetchedBrand.name || "Brand",
+    url: fetchedBrand?.url || brandInDb.url,
+    subdomain: fetchedBrand?.subdomain || brandInDb.subdomain,
     lastUpsertedTs: new Date(currentSyncDateMs),
   });
+
   return {
     helpCenterAllowed: brandInDb.helpCenterPermission === "read",
     ticketsAllowed: brandInDb.ticketsPermission === "read",
@@ -244,7 +198,7 @@ export async function getZendeskTicketsAllowedBrandIdsActivity(
 /**
  * Retrieves the categories for a given Brand.
  */
-export async function getZendeskCategoriesActivity({
+export async function fetchZendeskCategoriesActivity({
   connectorId,
   brandId,
 }: {
@@ -302,8 +256,7 @@ export async function syncZendeskCategoryActivity({
 
   // if all rights were revoked, we delete the category data.
   if (categoryInDb.permission === "none") {
-    await deleteCategoryChildren({ connectorId, dataSourceConfig, categoryId });
-    await categoryInDb.delete();
+    await deleteCategory({ connectorId, dataSourceConfig, categoryId });
     return false;
   }
 
@@ -319,8 +272,7 @@ export async function syncZendeskCategoryActivity({
   const { result: fetchedCategory } =
     await zendeskApiClient.helpcenter.categories.show(categoryId);
   if (!fetchedCategory) {
-    await deleteCategoryChildren({ connectorId, categoryId, dataSourceConfig });
-    await categoryInDb.delete();
+    await deleteCategory({ connectorId, categoryId, dataSourceConfig });
     return false;
   }
 
@@ -658,98 +610,3 @@ export async function syncZendeskTicketUpdateBatchActivity({
   );
   return { hasMore: !end_of_stream, afterCursor: after_cursor };
 }
-
-/**
- * Deletes all the tickets stored in the db and in the data source relative to a brand.
- */
-async function deleteBrandTickets({
-  connectorId,
-  brandId,
-  dataSourceConfig,
-}: {
-  connectorId: number;
-  brandId: number;
-  dataSourceConfig: DataSourceConfig;
-}) {
-  const tickets = await ZendeskTicketResource.fetchByBrandId({
-    connectorId,
-    brandId,
-  });
-  /// deleting the tickets in the data source
-  await Promise.all(
-    tickets.map((ticket) =>
-      deleteFromDataSource(
-        dataSourceConfig,
-        getTicketInternalId(ticket.connectorId, ticket.ticketId)
-      )
-    )
-  );
-  /// deleting the tickets stored in the db
-  await ZendeskTicketResource.deleteByBrandId({ connectorId, brandId });
-}
-
-/**
- * Deletes all the data stored in the db and in the data source relative to a brand's help center (category, articles).
- */
-async function deleteBrandHelpCenter({
-  connectorId,
-  brandId,
-  dataSourceConfig,
-}: {
-  connectorId: number;
-  brandId: number;
-  dataSourceConfig: DataSourceConfig;
-}) {
-  /// deleting the articles in the data source
-  const articles = await ZendeskArticleResource.fetchByBrandId({
-    connectorId,
-    brandId,
-  });
-  await Promise.all(
-    articles.map((article) =>
-      deleteFromDataSource(
-        dataSourceConfig,
-        getArticleInternalId(connectorId, article.articleId)
-      )
-    )
-  );
-  /// deleting the articles stored in the db
-  await ZendeskArticleResource.deleteByBrandId({
-    connectorId,
-    brandId,
-  });
-  /// deleting the categories stored in the db
-  await ZendeskCategoryResource.deleteByBrandId({ connectorId, brandId });
-}
-
-/**
- * Deletes all the data stored in the db and in the data source relative to a category (articles).
- */
-async function deleteCategoryChildren({
-  connectorId,
-  categoryId,
-  dataSourceConfig,
-}: {
-  connectorId: number;
-  categoryId: number;
-  dataSourceConfig: DataSourceConfig;
-}) {
-  /// deleting the articles in the data source
-  const articles = await ZendeskArticleResource.fetchByCategoryId({
-    connectorId,
-    categoryId,
-  });
-  await Promise.all(
-    articles.map((article) =>
-      deleteFromDataSource(
-        dataSourceConfig,
-        getArticleInternalId(connectorId, article.articleId)
-      )
-    )
-  );
-  /// deleting the articles stored in the db
-  await ZendeskArticleResource.deleteByCategoryId({
-    connectorId,
-    categoryId,
-  });
-}

connectors/src/connectors/zendesk/temporal/config.ts

@@ -1,4 +1,4 @@
-export const WORKFLOW_VERSION = 5;
+export const WORKFLOW_VERSION = 6;
 export const QUEUE_NAME = `zendesk-queue-v${WORKFLOW_VERSION}`;
 export const GARBAGE_COLLECT_QUEUE_NAME = `zendesk-gc-queue-v${WORKFLOW_VERSION}`;