Merge pull request #161 from canyongbs/feature/AIDAPP-144

[AIDAPP-144]: Reduce local queues and workers to only run a single instance

.env.example

@@ -55,10 +55,11 @@ AWS_SQS_ACCESS_KEY_ID=
 AWS_SQS_SECRET_ACCESS_KEY=
 SQS_PREFIX=
 SQS_QUEUE=default
-LANDLORD_SQS_QUEUE=landlord
-OUTBOUND_COMMUNICATION_QUEUE=outbound-communication
-AUDIT_QUEUE_QUEUE=audit
-IMPORT_EXPORT_QUEUE=import-export
+# Each of the following queues can be set to a different value if needed, but are set to the default for normal local development
+LANDLORD_SQS_QUEUE=default # or landlord
+OUTBOUND_COMMUNICATION_QUEUE=default # or outbound-communication
+AUDIT_QUEUE_QUEUE=default # or audit
+IMPORT_EXPORT_QUEUE=default # or import-export
 SQS_SUFFIX=
 AWS_SQS_DEFAULT_REGION=us-east-1
 
@@ -119,8 +120,12 @@ DEV_SUPER_ADMIN_EMAIL=
 DEV_USER_EMAILS=
 DEV_CONTACT_EMAILS=
 
-# Will specify the amount of queue worker processes to keep running when the container starts
-# Requires image rebuild if changed, will default to 3 if not set
+# If set to true queue workers will be created for each queue
+# When false it will only create queue workers for the SQS_QUEUE default queue
+# MULTIPLE_DEVELOPMENT_QUEUES=false
+
+# Will specify the amount of queue worker processes per queue to keep running when the container starts
+# Requires image rebuild if changed, will default to 3 if not set (will default to 1 in local dev development)
 #TOTAL_QUEUE_WORKERS=3
 
 ### END DEV SETTINGS ###

Dockerfile

@@ -40,8 +40,6 @@ ARG TOTAL_QUEUE_WORKERS=3
 
 COPY ./docker/generate-queues.sh /generate-queues.sh
 RUN chmod +x /generate-queues.sh
-RUN /generate-queues.sh
-RUN rm /generate-queues.sh
 
 COPY ./docker/nginx/nginx.conf /etc/nginx/nginx.conf
 COPY ./docker/nginx/site-opts.d /etc/nginx/site-opts.d
@@ -63,6 +61,18 @@ FROM base AS development
 #ARG GROUP_ID
 #RUN docker-php-serversideup-set-id www-data ${USER_ID} ${GROUP_ID}
 
+RUN if [[ -z "$MULTIPLE_DEVELOPMENT_QUEUES" ]] ; then \
+    /generate-queues.sh "default" "\$SQS_QUEUE" \
+    && /generate-queues.sh "landlord" "\$LANDLORD_SQS_QUEUE" \
+    && /generate-queues.sh "outbound-communication" "\$OUTBOUND_COMMUNICATION_QUEUE" \
+    && /generate-queues.sh "audit" "\$AUDIT_QUEUE_QUEUE" \
+    && /generate-queues.sh "import-export" "\$IMPORT_EXPORT_QUEUE" \
+    ; else \
+    /generate-queues.sh "default" "\$SQS_QUEUE" \
+    ; fi
+
+RUN rm /generate-queues.sh
+
 RUN chown -R "$PUID":"$PGID" /var/www/html \
     && if [[ -d /var/www/html/storage/logs ]] ; then \
     chgrp "$PGID" /var/www/html/storage/logs \
@@ -71,6 +81,14 @@ RUN chown -R "$PUID":"$PGID" /var/www/html \
 
 FROM base AS deploy
 
+RUN /generate-queues.sh "default" "\$SQS_QUEUE" \
+&& /generate-queues.sh "landlord" "\$LANDLORD_SQS_QUEUE" \
+&& /generate-queues.sh "outbound-communication" "\$OUTBOUND_COMMUNICATION_QUEUE" \
+&& /generate-queues.sh "audit" "\$AUDIT_QUEUE_QUEUE" \
+&& /generate-queues.sh "import-export" "\$IMPORT_EXPORT_QUEUE" 
+
+RUN rm /generate-queues.sh
+
 COPY --chown=$PUID:$PGID . /var/www/html
 
 RUN npm ci --ignore-scripts \

docker-compose.dev.yml

@@ -3,7 +3,8 @@ services:
     build:
       target: development
       args:
-        TOTAL_QUEUE_WORKERS: '${TOTAL_QUEUE_WORKERS:-3}'
+        TOTAL_QUEUE_WORKERS: '${TOTAL_QUEUE_WORKERS:-1}'
+        MULTIPLE_DEVELOPMENT_QUEUES: '${MULTIPLE_DEVELOPMENT_QUEUES:-false}'
     environment:
       SSL_MODE: "mixed"
       LANDLORD_MIGRATE: '${LANDLORD_MIGRATE:-true}'

docker/generate-queues.sh

@@ -1,25 +1,18 @@
 #!/bin/sh
 
-generate_services () {
-  QUEUE_NAME=$1
-  QUEUE_ENV_VAR=$2
+QUEUE_NAME=$1
+QUEUE_ENV_VAR=$2
 
-  for run in $(seq "$TOTAL_QUEUE_WORKERS"); do
-    if [ $run -eq 1 ]; then
-      SLEEP_TIME=0
-    else
-      SLEEP_TIME=3;
-    fi
+for run in $(seq "$TOTAL_QUEUE_WORKERS"); do
+# if [ $run -eq 1 ]; then
+#     SLEEP_TIME=0
+# else
+#     SLEEP_TIME=3;
+# fi
+SLEEP_TIME=3;
 
-    cp -r "/tmp/s6-overlay-templates/laravel-queue/service" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run";
-    sed -i -e "s/VAR_QUEUE/$QUEUE_ENV_VAR/g" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run/run";
-    sed -i -e "s/TEMPLATE_SLEEP/$SLEEP_TIME/g" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run/run";
-    cp "/tmp/s6-overlay-templates/laravel-queue/laravel-queue" "/etc/s6-overlay/s6-rc.d/user/contents.d/$QUEUE_NAME-queue-$run";
-  done
-}
-
-generate_services "default" "\$SQS_QUEUE"
-generate_services "landlord" "\$LANDLORD_SQS_QUEUE"
-generate_services "outbound-communication" "\$OUTBOUND_COMMUNICATION_QUEUE"
-generate_services "audit" "\$AUDIT_QUEUE_QUEUE"
-generate_services "import-export" "\$IMPORT_EXPORT_QUEUE"
+cp -r "/tmp/s6-overlay-templates/laravel-queue/service" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run";
+sed -i -e "s/VAR_QUEUE/$QUEUE_ENV_VAR/g" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run/run";
+sed -i -e "s/TEMPLATE_SLEEP/$SLEEP_TIME/g" "/etc/s6-overlay/s6-rc.d/$QUEUE_NAME-queue-$run/run";
+cp "/tmp/s6-overlay-templates/laravel-queue/laravel-queue" "/etc/s6-overlay/s6-rc.d/user/contents.d/$QUEUE_NAME-queue-$run";
+done