Init new members enrichment worker

.github/workflows/lf-staging-deploy-members-enrichment-worker.yaml

@@ -0,0 +1,60 @@
+name: LF Staging Deploy Members Enrichment Worker
+
+on:
+  push:
+    branches:
+      - 'lf-staging/**'
+      - 'lf-staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/premium/members_enrichment_worker/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.LF_STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.LF_STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.LF_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.LF_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.LF_AWS_REGION }}
+  SLACK_CHANNEL: deploys-lf-staging
+  SLACK_WEBHOOK: ${{ secrets.LF_STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: members-enrichment-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-members-enrichment-worker:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: members-enrichment-worker
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/staging-deploy-members-enrichment-worker.yaml

@@ -0,0 +1,60 @@
+name: Staging Deploy Members Enrichment Worker
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+      - 'staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/premium/members_enrichment_worker/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  SLACK_CHANNEL: deploys-staging
+  SLACK_WEBHOOK: ${{ secrets.STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: members-enrichment-worker
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-members-enrichment-worker:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: members-enrichment-worker
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

backend/package.json

@@ -128,7 +128,7 @@
     "passport-google-oauth": "2.0.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-slack": "0.0.7",
-    "peopledatalabs": "^5.0.3",
+    "peopledatalabs": "~6.1.5",
     "pg": "^8.7.3",
     "pm2": "^5.2.0",
     "sanitize-html": "^2.7.1",

backend/src/services/premium/enrichment/memberEnrichmentService.ts

@@ -15,14 +15,6 @@ import {
   SyncMode,
   IOrganizationIdentity,
 } from '@crowd/types'
-import { ENRICHMENT_CONFIG, REDIS_CONFIG } from '../../../conf'
-import { AttributeData } from '../../../database/attributes/attribute'
-import MemberEnrichmentCacheRepository from '../../../database/repositories/memberEnrichmentCacheRepository'
-import track from '../../../segment/track'
-import { Member } from '../../../serverless/integrations/types/messageTypes'
-import { IServiceOptions } from '../../IServiceOptions'
-import MemberAttributeSettingsService from '../../memberAttributeSettingsService'
-import MemberService from '../../memberService'
 import {
   EnrichmentAPICertification,
   EnrichmentAPIContribution,
@@ -31,7 +23,15 @@ import {
   EnrichmentAPIResponse,
   EnrichmentAPISkills,
   EnrichmentAPIWorkExperience,
-} from './types/memberEnrichmentTypes'
+} from '@crowd/types/premium'
+import { ENRICHMENT_CONFIG, REDIS_CONFIG } from '../../../conf'
+import { AttributeData } from '../../../database/attributes/attribute'
+import MemberEnrichmentCacheRepository from '../../../database/repositories/memberEnrichmentCacheRepository'
+import track from '../../../segment/track'
+import { Member } from '../../../serverless/integrations/types/messageTypes'
+import { IServiceOptions } from '../../IServiceOptions'
+import MemberAttributeSettingsService from '../../memberAttributeSettingsService'
+import MemberService from '../../memberService'
 import OrganizationService from '../../organizationService'
 import MemberRepository from '../../../database/repositories/memberRepository'
 import OrganizationRepository from '../../../database/repositories/organizationRepository'

backend/src/services/premium/enrichment/organizationEnrichmentService.ts

@@ -11,11 +11,11 @@ import {
   PlatformType,
   SyncMode,
 } from '@crowd/types'
+import { EnrichmentParams, IEnrichmentResponse } from '@crowd/types/premium'
 import { REDIS_CONFIG } from '../../../conf'
 import OrganizationRepository from '../../../database/repositories/organizationRepository'
 import { renameKeys } from '../../../utils/renameKeys'
 import { IServiceOptions } from '../../IServiceOptions'
-import { EnrichmentParams, IEnrichmentResponse } from './types/organizationEnrichmentTypes'
 import SequelizeRepository from '@/database/repositories/sequelizeRepository'
 import SearchSyncService from '@/services/searchSyncService'
 

backend/tsconfig.json

@@ -13,6 +13,7 @@
     "types": ["node", "jest"],
     "baseUrl": "./src",
     "paths": {
+      "@crowd/*/premium": ["../../services/libs/*/src/premium"],
       "@crowd/*": ["../../services/libs/*/src"],
       "@/*": ["./*"]
     },

pnpm-workspace.yaml

@@ -2,4 +2,5 @@ packages:
   - 'services/libs/*'
   - 'services/archetypes/*'
   - 'services/apps/*'
+  - 'services/apps/premium/*'
   - 'backend'

scripts/builders/members-enrichment-worker.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+DOCKERFILE="./services/docker/Dockerfile.members_enrichment_worker"
+CONTEXT="../"
+REPO="crowddotdev/members-enrichment-worker"

scripts/scaffold.yaml

@@ -154,12 +154,23 @@ services:
     build:
       context: scaffold/temporal
     restart: always
+    entrypoint: ["temporal", "server", "start-dev", "--ip", "0.0.0.0"]
     ports:
       - '7233:7233'
       - '8233:8233'
     networks:
       - crowd-bridge
 
+  temporal-setup:
+    build:
+      context: scaffold/temporal
+    restart: no
+    depends_on:
+      - temporal
+    command: "temporal operator search-attribute create --name TenantId --type Text --address temporal:7233"
+    networks:
+      - crowd-bridge
+
 networks:
   crowd-bridge:
     external: true