Merge branch 'main' into feat/new-groupsio-connect

CrowdDotDev · Oct 7, 2024 · 869d891 · 869d891
2 parents 71e6ce1 + 425e2d2
commit 869d891
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 60 deletions.
diff --git a/backend/src/services/memberService.ts b/backend/src/services/memberService.ts
@@ -53,7 +53,7 @@ import {
   removeMemberTasks,
 } from '@crowd/data-access-layer/src/members'
 import { findMemberAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations'
-import { getActivityCountOfMemberIdentities } from '@crowd/data-access-layer'
+// import { getActivityCountOfMemberIdentities } from '@crowd/data-access-layer'
 import { fetchManySegments } from '@crowd/data-access-layer/src/segments'
 import { QueryExecutor, optionsQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { TEMPORAL_CONFIG } from '@/conf'
@@ -1161,17 +1161,19 @@ export default class MemberService extends LoggerBase {
           )
           member.memberOrganizations = unmergedRoles as IMemberRoleWithOrganization[]
 
+          const secondaryActivityCount = 0
+          const primaryActivityCount = 0
           // activity count
-          const secondaryActivityCount = await getActivityCountOfMemberIdentities(
-            this.options.qdb,
-            member.id,
-            secondaryBackup.identities,
-          )
-          const primaryActivityCount = await getActivityCountOfMemberIdentities(
-            this.options.qdb,
-            member.id,
-            member.identities,
-          )
+          // const secondaryActivityCount = await getActivityCountOfMemberIdentities(
+          //   this.options.qdb,
+          //   member.id,
+          //   secondaryBackup.identities,
+          // )
+          // const primaryActivityCount = await getActivityCountOfMemberIdentities(
+          //   this.options.qdb,
+          //   member.id,
+          //   member.identities,
+          // )
 
           return {
             primary: {
@@ -1210,17 +1212,20 @@ export default class MemberService extends LoggerBase {
         throw new Error(`Original member only has one identity, cannot extract it!`)
       }
 
-      const secondaryActivityCount = await getActivityCountOfMemberIdentities(
-        this.options.qdb,
-        member.id,
-        secondaryIdentities,
-      )
-
-      const primaryActivityCount = await getActivityCountOfMemberIdentities(
-        this.options.qdb,
-        member.id,
-        primaryIdentities,
-      )
+      const secondaryActivityCount = 0
+      const primaryActivityCount = 0
+
+      // const secondaryActivityCount = await getActivityCountOfMemberIdentities(
+      //   this.options.qdb,
+      //   member.id,
+      //   secondaryIdentities,
+      // )
+      //
+      // const primaryActivityCount = await getActivityCountOfMemberIdentities(
+      //   this.options.qdb,
+      //   member.id,
+      //   primaryIdentities,
+      // )
 
       const primaryMemberRoles = await MemberOrganizationRepository.findMemberRoles(
         member.id,

diff --git a/frontend/src/modules/contributor/components/shared/contributor-last-enrichment.vue b/frontend/src/modules/contributor/components/shared/contributor-last-enrichment.vue
@@ -3,18 +3,11 @@
     <p class="text-tiny text-gray-500 mr-2 text-right">
       Last enrichment on {{ moment(props.contributor.lastEnriched).format('MMM DD, YYYY') }}
     </p>
-    <lf-tooltip
-      content="Profile is enriched via GitHub identity or email. Enrichment is processed every 3 months."
-    >
-      <lf-icon name="question-line" :size="16" class="text-gray-400" />
-    </lf-tooltip>
   </div>
 </template>
 
 <script setup lang="ts">
-import LfIcon from '@/ui-kit/icon/Icon.vue';
 import moment from 'moment';
-import LfTooltip from '@/ui-kit/tooltip/Tooltip.vue';
 import { Contributor } from '@/modules/contributor/types/Contributor';
 
 const props = defineProps<{

diff --git a/frontend/src/modules/organization/components/shared/organization-last-enrichment.vue b/frontend/src/modules/organization/components/shared/organization-last-enrichment.vue
@@ -3,18 +3,11 @@
     <p class="text-tiny text-gray-500 mr-2 text-right">
       Last enrichment on {{ moment(props.organization.lastEnrichedAt).format('MMM DD, YYYY') }}
     </p>
-    <lf-tooltip
-      content="Organization data is enriched via GitHub identity. Enrichment is processed every 3 months."
-    >
-      <lf-icon name="question-line" :size="16" class="text-gray-400" />
-    </lf-tooltip>
   </div>
 </template>
 
 <script setup lang="ts">
-import LfIcon from '@/ui-kit/icon/Icon.vue';
 import moment from 'moment';
-import LfTooltip from '@/ui-kit/tooltip/Tooltip.vue';
 import { Organization } from '@/modules/organization/types/Organization';
 
 const props = defineProps<{

diff --git a/services/apps/data_sink_worker/src/service/activity.service.ts b/services/apps/data_sink_worker/src/service/activity.service.ts
@@ -244,27 +244,62 @@ export default class ActivityService extends LoggerBase {
             platform: toUpdate.platform || (original.platform as PlatformType),
           })
 
-          await updateActivity(this.qdbStore.connection(), id, {
-            tenantId: tenantId,
-            segmentId: segmentId,
-            type: toUpdate.type || original.type,
-            isContribution: toUpdate.isContribution || original.isContribution,
-            score: toUpdate.score || original.score,
-            sourceId: toUpdate.sourceId || original.sourceId,
-            sourceParentId: toUpdate.sourceParentId || original.sourceParentId,
-            memberId: toUpdate.memberId || original.memberId,
-            username: toUpdate.username || original.username,
-            sentiment: toUpdate.sentiment || original.sentiment,
-            attributes: toUpdate.attributes || original.attributes,
-            body: escapeNullByte(toUpdate.body || original.body),
-            title: escapeNullByte(toUpdate.title || original.title),
-            channel: toUpdate.channel || original.channel,
-            url: toUpdate.url || original.url,
-            organizationId: toUpdate.organizationId || original.organizationId,
-            platform: toUpdate.platform || (original.platform as PlatformType),
-            isBotActivity: memberInfo.isBot,
-            isTeamMemberActivity: memberInfo.isTeamMember,
-          })
+          // use insert instead of update to avoid using pg protocol with questdb
+          try {
+            await insertActivities([
+              {
+                id,
+                memberId: toUpdate.memberId || original.memberId,
+                timestamp: original.timestamp,
+                platform: toUpdate.platform || (original.platform as PlatformType),
+                type: toUpdate.type || original.type,
+                isContribution: toUpdate.isContribution || original.isContribution,
+                score: toUpdate.score || original.score,
+                sourceId: toUpdate.sourceId || original.sourceId,
+                sourceParentId: toUpdate.sourceParentId || original.sourceParentId,
+                tenantId: tenantId,
+                attributes: toUpdate.attributes || original.attributes,
+                sentiment: toUpdate.sentiment || original.sentiment,
+                body: escapeNullByte(toUpdate.body || original.body),
+                title: escapeNullByte(toUpdate.title || original.title),
+                channel: toUpdate.channel || original.channel,
+                url: toUpdate.url || original.url,
+                username: toUpdate.username || original.username,
+                objectMemberId: activity.objectMemberId,
+                objectMemberUsername: activity.objectMemberUsername,
+                segmentId: segmentId,
+                organizationId: toUpdate.organizationId || original.organizationId,
+                isBotActivity: memberInfo.isBot,
+                isTeamMemberActivity: memberInfo.isTeamMember,
+                importHash: original.importHash,
+              },
+            ])
+          } catch (error) {
+            this.log.error('Error updating (by inserting) activity in QuestDB:', error)
+            throw error
+          }
+
+          // await updateActivity(this.qdbStore.connection(), id, {
+          //   tenantId: tenantId,
+          //   segmentId: segmentId,
+          //   type: toUpdate.type || original.type,
+          //   isContribution: toUpdate.isContribution || original.isContribution,
+          //   score: toUpdate.score || original.score,
+          //   sourceId: toUpdate.sourceId || original.sourceId,
+          //   sourceParentId: toUpdate.sourceParentId || original.sourceParentId,
+          //   memberId: toUpdate.memberId || original.memberId,
+          //   username: toUpdate.username || original.username,
+          //   sentiment: toUpdate.sentiment || original.sentiment,
+          //   attributes: toUpdate.attributes || original.attributes,
+          //   body: escapeNullByte(toUpdate.body || original.body),
+          //   title: escapeNullByte(toUpdate.title || original.title),
+          //   channel: toUpdate.channel || original.channel,
+          //   url: toUpdate.url || original.url,
+          //   organizationId: toUpdate.organizationId || original.organizationId,
+          //   platform: toUpdate.platform || (original.platform as PlatformType),
+          //   isBotActivity: memberInfo.isBot,
+          //   isTeamMemberActivity: memberInfo.isTeamMember,
+          // })
 
           return true
         } else {

diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/activity.data.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/activity.data.ts
@@ -17,6 +17,7 @@ export interface IDbActivity {
   objectMemberId?: string
   objectMemberUsername?: string
   attributes: Record<string, unknown>
+  importHash?: string
   body?: string
   title?: string
   channel?: string

diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/activity.repo.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/activity.repo.ts
@@ -35,6 +35,7 @@ export default class ActivityRepository extends RepositoryBase<ActivityRepositor
             "objectMemberId",
             "objectMemberUsername",
             attributes,
+            "importHash",
             body,
             title,
             channel,
@@ -92,6 +93,7 @@ export default class ActivityRepository extends RepositoryBase<ActivityRepositor
               "objectMemberId",
               "objectMemberUsername",
               attributes,
+              "importHash",
               body,
               title,
               channel,

diff --git a/services/libs/database/src/connection.ts b/services/libs/database/src/connection.ts
@@ -1,7 +1,7 @@
 import { getServiceChildLogger } from '@crowd/logging'
 import pgPromise from 'pg-promise'
 import { DbConnection, DbInstance, IDatabaseConfig } from './types'
-import { IS_CLOUD_ENV } from '@crowd/common'
+import { IS_CLOUD_ENV, IS_DEV_ENV } from '@crowd/common'
 
 const log = getServiceChildLogger('database.connection')
 
@@ -81,7 +81,7 @@ export const getDbConnection = async (
           rejectUnauthorized: false,
         }
       : false,
-    max: maxPoolSize || 20,
+    max: maxPoolSize || (IS_DEV_ENV ? 5 : 20),
     idleTimeoutMillis: idleTimeoutMillis !== undefined ? idleTimeoutMillis : 10000,
     // query_timeout: 30000,
     application_name: process.env.SERVICE || 'unknown-app',

diff --git a/services/libs/integrations/src/integrations/github/api/graphql/pullRequests.ts b/services/libs/integrations/src/integrations/github/api/graphql/pullRequests.ts
@@ -5,7 +5,7 @@ import BaseQuery from './baseQuery'
 class PullRequestsQuery extends BaseQuery {
   repo: Repo
 
-  constructor(repo: Repo, githubToken: string, perPage = 20) {
+  constructor(repo: Repo, githubToken: string, perPage = 5) {
     const pullRequestsQuery = `{
             repository(owner: "${repo.owner}", name: "${repo.name}") {
               pullRequests(last: ${perPage}, \${beforeCursor}) {