From 43286f96f60c537f32d96d68e7a5abd83090767f Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 11:26:04 +0300
Subject: [PATCH 1/7] Fix discord error handling (#1726)

---
 .../integrations/discord/api/getMembers.ts    | 41 +++++++---------
 .../integrations/discord/api/getMessages.ts   | 48 ++++++++-----------
 2 files changed, 38 insertions(+), 51 deletions(-)

diff --git a/services/libs/integrations/src/integrations/discord/api/getMembers.ts b/services/libs/integrations/src/integrations/discord/api/getMembers.ts
index 80f62ca44e..e72820b661 100644
--- a/services/libs/integrations/src/integrations/discord/api/getMembers.ts
+++ b/services/libs/integrations/src/integrations/discord/api/getMembers.ts
@@ -2,25 +2,27 @@ import axios from 'axios'
 import { DiscordApiMember, DiscordGetMembersInput, DiscordGetMembersOutput } from '../types'
 import { IProcessStreamContext } from '../../../types'
 import { getRateLimiter } from './handleRateLimit'
+import { handleDiscordError } from './errorHandler'
 
 async function getMembers(
   input: DiscordGetMembersInput,
   ctx: IProcessStreamContext,
 ): Promise<DiscordGetMembersOutput> {
   const rateLimiter = getRateLimiter(ctx)
-  try {
-    let url = `https://discord.com/api/v10/guilds/${input.guildId}/members?limit=${input.perPage}`
-    if (input.page !== undefined && input.page !== '') {
-      url += `&after=${input.page}`
-    }
-    const config = {
-      method: 'get',
-      url,
-      headers: {
-        Authorization: input.token,
-      },
-    }
 
+  let url = `https://discord.com/api/v10/guilds/${input.guildId}/members?limit=${input.perPage}`
+  if (input.page !== undefined && input.page !== '') {
+    url += `&after=${input.page}`
+  }
+
+  const config = {
+    method: 'get',
+    url,
+    headers: {
+      Authorization: input.token,
+    },
+  }
+  try {
     await rateLimiter.checkRateLimit('getMembers')
     await rateLimiter.incrementRateLimit()
     const response = await axios(config)
@@ -35,19 +37,10 @@ async function getMembers(
       timeUntilReset,
     }
   } catch (err) {
-    if (err.response.status === 429) {
-      ctx.log.warn(
-        `Rate limit exceeded in Get Members. Wait value in header is ${err.response.headers['x-ratelimit-reset-after']}`,
-      )
-      return {
-        records: [],
-        nextPage: input.page,
-        limit: 0,
-        timeUntilReset: err.response.headers['x-ratelimit-reset-after'],
-      }
+    const newErr = handleDiscordError(err, config, { input }, ctx)
+    if (newErr) {
+      throw newErr
     }
-    ctx.log.error({ err, input }, 'Error while getting members from Discord')
-    throw err
   }
 }
 
diff --git a/services/libs/integrations/src/integrations/discord/api/getMessages.ts b/services/libs/integrations/src/integrations/discord/api/getMessages.ts
index 19e8bc98d7..2f6c2fd201 100644
--- a/services/libs/integrations/src/integrations/discord/api/getMessages.ts
+++ b/services/libs/integrations/src/integrations/discord/api/getMessages.ts
@@ -2,26 +2,29 @@ import axios from 'axios'
 import { DiscordApiMessage, DiscordParsedReponse, DiscordGetMessagesInput } from '../types'
 import { IProcessStreamContext } from '../../../types'
 import { getRateLimiter } from './handleRateLimit'
+import { handleDiscordError } from './errorHandler'
 
 async function getMessages(
   input: DiscordGetMessagesInput,
   ctx: IProcessStreamContext,
-  showError = true,
+  showErrors = true,
 ): Promise<DiscordParsedReponse> {
   const rateLimiter = getRateLimiter(ctx)
-  try {
-    let url = `https://discord.com/api/v10/channels/${input.channelId}/messages?limit=${input.perPage}`
-    if (input.page !== undefined && input.page !== '') {
-      url += `&before=${input.page}`
-    }
-    const config = {
-      method: 'get',
-      url,
-      headers: {
-        Authorization: input.token,
-      },
-    }
 
+  let url = `https://discord.com/api/v10/channels/${input.channelId}/messages?limit=${input.perPage}`
+  if (input.page !== undefined && input.page !== '') {
+    url += `&before=${input.page}`
+  }
+
+  const config = {
+    method: 'get',
+    url,
+    headers: {
+      Authorization: input.token,
+    },
+  }
+
+  try {
     await rateLimiter.checkRateLimit('getMessages')
     await rateLimiter.incrementRateLimit()
     const response = await axios(config)
@@ -37,18 +40,7 @@ async function getMessages(
       timeUntilReset,
     }
   } catch (err) {
-    if (err.response.status === 429) {
-      ctx.log.warn(
-        `Rate limit exceeded in Get Messages. Wait value in header is ${err.response.headers['x-ratelimit-reset-after']}`,
-      )
-      return {
-        records: [],
-        nextPage: input.page,
-        limit: 0,
-        timeUntilReset: err.response.headers['x-ratelimit-reset-after'],
-      }
-    }
-    if (!showError) {
+    if (!showErrors) {
       return {
         records: [],
         nextPage: '',
@@ -56,8 +48,10 @@ async function getMessages(
         timeUntilReset: 0,
       }
     }
-    ctx.log.error({ err, input }, 'Error while getting messages from Discord')
-    throw err
+    const newErr = handleDiscordError(err, config, { input }, ctx)
+    if (newErr) {
+      throw newErr
+    }
   }
 }
 

From fa2817faaa749558c244d5fbd2c20167cedce201 Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 15:15:18 +0300
Subject: [PATCH 2/7] Fix rate limiter and increase requests for Discord
 (#1729)

---
 .../src/integrations/discord/api/handleRateLimit.ts         | 4 ++--
 services/libs/redis/src/rateLimiter.ts                      | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
index 9b4ae6f694..2b34e17544 100644
--- a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
+++ b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
@@ -1,7 +1,7 @@
 import { IProcessStreamContext } from '../../../types'
 
-const DISCORD_RATE_LIMIT = 50
-const DISCORD_RATE_LIMIT_TIME = 1
+const DISCORD_RATE_LIMIT = 100
+const DISCORD_RATE_LIMIT_TIME = 1 // 1 second
 const REDIS_KEY = 'discord-request-count'
 
 export const getRateLimiter = (ctx: IProcessStreamContext) => {
diff --git a/services/libs/redis/src/rateLimiter.ts b/services/libs/redis/src/rateLimiter.ts
index 411b4b65e6..c630d9a083 100644
--- a/services/libs/redis/src/rateLimiter.ts
+++ b/services/libs/redis/src/rateLimiter.ts
@@ -19,6 +19,10 @@ export class RateLimiter implements IRateLimiter {
     const requestCount = value === null ? 0 : parseInt(value)
     const canMakeRequest = requestCount < this.maxRequests
 
+    if (requestCount === 0) {
+      await this.cache.set(this.counterKey, '0', this.timeWindowSeconds)
+    }
+
     if (!canMakeRequest) {
       const sleepTime = this.timeWindowSeconds + Math.floor(Math.random() * this.maxRequests)
       throw new RateLimitError(sleepTime, endpoint)
@@ -26,7 +30,7 @@ export class RateLimiter implements IRateLimiter {
   }
 
   public async incrementRateLimit() {
-    await this.cache.increment(this.counterKey, 1, this.timeWindowSeconds)
+    await this.cache.increment(this.counterKey, 1)
   }
 }
 

From da8839ef5b70467c34b2fd3a94d7333feb078653 Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 16:05:36 +0300
Subject: [PATCH 3/7] Change rate limit configuration for Discord (#1730)

---
 .../src/integrations/discord/api/handleRateLimit.ts         | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
index 2b34e17544..f09faad4ef 100644
--- a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
+++ b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
@@ -1,8 +1,8 @@
 import { IProcessStreamContext } from '../../../types'
 
-const DISCORD_RATE_LIMIT = 100
-const DISCORD_RATE_LIMIT_TIME = 1 // 1 second
-const REDIS_KEY = 'discord-request-count'
+const DISCORD_RATE_LIMIT = 10000
+const DISCORD_RATE_LIMIT_TIME = 100 // 100 seconds
+const REDIS_KEY = 'discord-ratelimits-requests-count'
 
 export const getRateLimiter = (ctx: IProcessStreamContext) => {
   return ctx.getRateLimiter(DISCORD_RATE_LIMIT, DISCORD_RATE_LIMIT_TIME, REDIS_KEY)

From b5c6d24dec31c02172306739d9120e2cbaa8c0d2 Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 18:40:14 +0300
Subject: [PATCH 4/7] Update configuration for Discord integration (#1731)

---
 .../src/integrations/discord/api/handleRateLimit.ts           | 2 +-
 services/libs/integrations/src/integrations/discord/index.ts  | 2 +-
 .../integrations/src/integrations/discord/processStream.ts    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
index f09faad4ef..cf66967c7a 100644
--- a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
+++ b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts
@@ -1,6 +1,6 @@
 import { IProcessStreamContext } from '../../../types'
 
-const DISCORD_RATE_LIMIT = 10000
+const DISCORD_RATE_LIMIT = 100000
 const DISCORD_RATE_LIMIT_TIME = 100 // 100 seconds
 const REDIS_KEY = 'discord-ratelimits-requests-count'
 
diff --git a/services/libs/integrations/src/integrations/discord/index.ts b/services/libs/integrations/src/integrations/discord/index.ts
index 1309e53576..a8806244d9 100644
--- a/services/libs/integrations/src/integrations/discord/index.ts
+++ b/services/libs/integrations/src/integrations/discord/index.ts
@@ -9,7 +9,7 @@ import processWebhookStream from './processWebhookStream'
 const descriptor: IIntegrationDescriptor = {
   type: PlatformType.DISCORD,
   memberAttributes: DISCORD_MEMBER_ATTRIBUTES,
-  checkEvery: 4 * 60, // 4 hours
+  checkEvery: 12 * 60, // 12 hours
   generateStreams,
   processStream,
   processData,
diff --git a/services/libs/integrations/src/integrations/discord/processStream.ts b/services/libs/integrations/src/integrations/discord/processStream.ts
index 68524c721e..b92cc7ae0e 100644
--- a/services/libs/integrations/src/integrations/discord/processStream.ts
+++ b/services/libs/integrations/src/integrations/discord/processStream.ts
@@ -202,7 +202,7 @@ const processMembersStream: ProcessStreamHandler = async (ctx) => {
       guildId: data.guildId,
       token: getDiscordToken(ctx),
       page: data.page,
-      perPage: 100,
+      perPage: 500,
     },
     ctx,
   )
@@ -241,7 +241,7 @@ const processChannelStream: ProcessStreamHandler = async (ctx) => {
       channelId: data.channelId,
       token: getDiscordToken(ctx),
       page: data.page,
-      perPage: 100,
+      perPage: 500,
     },
     ctx,
   )

From ed08acb5647cd8960f24f85da3bceca21e0a1a2e Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 18:47:45 +0300
Subject: [PATCH 5/7] Fix sleeping time in rate limiter (#1732)

---
 services/libs/redis/src/rateLimiter.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/libs/redis/src/rateLimiter.ts b/services/libs/redis/src/rateLimiter.ts
index c630d9a083..ffb9e42bad 100644
--- a/services/libs/redis/src/rateLimiter.ts
+++ b/services/libs/redis/src/rateLimiter.ts
@@ -24,7 +24,7 @@ export class RateLimiter implements IRateLimiter {
     }
 
     if (!canMakeRequest) {
-      const sleepTime = this.timeWindowSeconds + Math.floor(Math.random() * this.maxRequests)
+      const sleepTime = this.timeWindowSeconds + Math.floor(Math.random() * this.timeWindowSeconds)
       throw new RateLimitError(sleepTime, endpoint)
     }
   }

From e2ed88e340af8e85618a903ea7520f9659ac09af Mon Sep 17 00:00:00 2001
From: Igor Kotua <36304232+garrrikkotua@users.noreply.github.com>
Date: Thu, 19 Oct 2023 18:58:31 +0300
Subject: [PATCH 6/7] Fix per page in Discord integration (#1733)

---
 .../integrations/src/integrations/discord/processStream.ts    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/libs/integrations/src/integrations/discord/processStream.ts b/services/libs/integrations/src/integrations/discord/processStream.ts
index b92cc7ae0e..5a3c82d549 100644
--- a/services/libs/integrations/src/integrations/discord/processStream.ts
+++ b/services/libs/integrations/src/integrations/discord/processStream.ts
@@ -202,7 +202,7 @@ const processMembersStream: ProcessStreamHandler = async (ctx) => {
       guildId: data.guildId,
       token: getDiscordToken(ctx),
       page: data.page,
-      perPage: 500,
+      perPage: 1000,
     },
     ctx,
   )
@@ -241,7 +241,7 @@ const processChannelStream: ProcessStreamHandler = async (ctx) => {
       channelId: data.channelId,
       token: getDiscordToken(ctx),
       page: data.page,
-      perPage: 500,
+      perPage: 100,
     },
     ctx,
   )

From 1b21c9f1dda5a588d9d3409b48905b5ce741717e Mon Sep 17 00:00:00 2001
From: anilb <epipav@gmail.com>
Date: Fri, 20 Oct 2023 09:48:55 +0200
Subject: [PATCH 7/7] Query fix for org sync in search sync worker (#1736)

---
 services/apps/search_sync_worker/src/repo/organization.repo.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/services/apps/search_sync_worker/src/repo/organization.repo.ts b/services/apps/search_sync_worker/src/repo/organization.repo.ts
index 9c8031ef43..8c2f3f97e6 100644
--- a/services/apps/search_sync_worker/src/repo/organization.repo.ts
+++ b/services/apps/search_sync_worker/src/repo/organization.repo.ts
@@ -146,6 +146,8 @@ export class OrganizationRepository extends RepositoryBase<OrganizationRepositor
         FROM organizations o
         LEFT JOIN member_data md ON o.id = md."organizationId"
         LEFT JOIN identities i ON o.id = i."organizationId"
+        LEFT JOIN to_merge_data tmd on o.id = tmd."organizationId"
+        LEFT JOIN no_merge_data nmd on o.id = nmd."organizationId"
         WHERE o.id IN ($(ids:csv))
           AND o."deletedAt" IS NULL
           AND (md."organizationId" IS NOT NULL