From 118953e29a99f261a600ee557a9e6cc7db37d031 Mon Sep 17 00:00:00 2001
From: Aric Lasry <aric@dust.tt>
Date: Wed, 10 Apr 2024 14:32:57 +0200
Subject: [PATCH] Starting script + increase server size (#4653)

---
 connectors/package.json                       |  1 +
 connectors/src/start_all_workers.sh           | 22 +++++++++++++++++++
 .../connectors-worker-deployment.yaml         | 18 ++++-----------
 3 files changed, 27 insertions(+), 14 deletions(-)
 create mode 100755 connectors/src/start_all_workers.sh

diff --git a/connectors/package.json b/connectors/package.json
index b3b427591de2..d16e9701fe21 100644
--- a/connectors/package.json
+++ b/connectors/package.json
@@ -9,6 +9,7 @@
     "start": "tsx ./src/start.ts -p 3002",
     "start:web": "tsx ./src/start_server.ts -p 3002",
     "start:worker": "tsx ./src/start_worker.ts",
+    "start:worker:processes": "./src/start_all_workers.sh",
     "cli": "npx tsx src/admin/cli.ts",
     "initdb": "npx tsx src/admin/db.ts"
   },
diff --git a/connectors/src/start_all_workers.sh b/connectors/src/start_all_workers.sh
new file mode 100755
index 000000000000..fcd02291e3ff
--- /dev/null
+++ b/connectors/src/start_all_workers.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Bash scripts that starts all the workers in the background
+# and fails when any of them fails.
+# The return code of the first failing worker is returned 
+# as the return code of this script.
+
+fail() {
+    wait -n
+    exit $?
+}
+
+npm run start:worker -- --workers confluence & 
+npm run start:worker -- --workers github & 
+npm run start:worker -- --workers google_drive & 
+npm run start:worker -- --workers intercom & 
+# npm run start:worker -- --workers notion &  // notion is running on its own pod so no need to run it here
+npm run start:worker -- --workers slack &
+npm run start:worker -- --workers webcrawler & 
+
+
+fail
\ No newline at end of file
diff --git a/k8s/deployments/connectors-worker-deployment.yaml b/k8s/deployments/connectors-worker-deployment.yaml
index 1ebf209c95e3..7a6e65907654 100644
--- a/k8s/deployments/connectors-worker-deployment.yaml
+++ b/k8s/deployments/connectors-worker-deployment.yaml
@@ -19,18 +19,8 @@ spec:
       containers:
         - name: web
           image: gcr.io/or1g1n-186209/connectors-image:latest
-          command: ["npm", "run", "start:worker"]
-          args:
-            [
-              "--",
-              "--workers",
-              "confluence",
-              "github",
-              "google_drive",
-              "intercom",
-              "slack",
-              "webcrawler",
-            ]
+          command: ["npm", "run", "start:worker:processes"]
+          args: []
           imagePullPolicy: Always
           envFrom:
             - configMapRef:
@@ -53,12 +43,12 @@ spec:
           resources:
             requests:
               cpu: 3000m
-              memory: 8Gi
+              memory: 32Gi
               ephemeral-storage: 32Gi
 
             limits:
               cpu: 3000m
-              memory: 8Gi
+              memory: 32Gi
               ephemeral-storage: 32Gi
 
       volumes: