Starting script + increase server size (#4653)

dust-tt · Apr 10, 2024 · 118953e · 118953e
1 parent 89f969f
commit 118953e
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 14 deletions.
diff --git a/connectors/package.json b/connectors/package.json
@@ -9,6 +9,7 @@
     "start": "tsx ./src/start.ts -p 3002",
     "start:web": "tsx ./src/start_server.ts -p 3002",
     "start:worker": "tsx ./src/start_worker.ts",
+    "start:worker:processes": "./src/start_all_workers.sh",
     "cli": "npx tsx src/admin/cli.ts",
     "initdb": "npx tsx src/admin/db.ts"
   },

diff --git a/connectors/src/start_all_workers.sh b/connectors/src/start_all_workers.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Bash scripts that starts all the workers in the background
+# and fails when any of them fails.
+# The return code of the first failing worker is returned 
+# as the return code of this script.
+
+fail() {
+    wait -n
+    exit $?
+}
+
+npm run start:worker -- --workers confluence & 
+npm run start:worker -- --workers github & 
+npm run start:worker -- --workers google_drive & 
+npm run start:worker -- --workers intercom & 
+# npm run start:worker -- --workers notion &  // notion is running on its own pod so no need to run it here
+npm run start:worker -- --workers slack &
+npm run start:worker -- --workers webcrawler & 
+
+
+fail
diff --git a/k8s/deployments/connectors-worker-deployment.yaml b/k8s/deployments/connectors-worker-deployment.yaml
@@ -19,18 +19,8 @@ spec:
       containers:
         - name: web
           image: gcr.io/or1g1n-186209/connectors-image:latest
-          command: ["npm", "run", "start:worker"]
-          args:
-            [
-              "--",
-              "--workers",
-              "confluence",
-              "github",
-              "google_drive",
-              "intercom",
-              "slack",
-              "webcrawler",
-            ]
+          command: ["npm", "run", "start:worker:processes"]
+          args: []
           imagePullPolicy: Always
           envFrom:
             - configMapRef:
@@ -53,12 +43,12 @@ spec:
           resources:
             requests:
               cpu: 3000m
-              memory: 8Gi
+              memory: 32Gi
               ephemeral-storage: 32Gi
 
             limits:
               cpu: 3000m
-              memory: 8Gi
+              memory: 32Gi
               ephemeral-storage: 32Gi
 
       volumes: