Merge pull request #2 from eHealthAfrica/wip/translate-to-jobs

translate to jobs consumer
eHealthAfrica · Feb 20, 2020 · 367670f · 367670f
2 parents cfc24b4 + d19c98b
commit 367670f
Show file tree

Hide file tree

Showing 30 changed files with 2,572 additions and 705 deletions.
diff --git a/.gitmodules b/.gitmodules
diff --git a/aether-bootstrap b/aether-bootstrap
diff --git a/conf/test-firebase.json b/conf/test-firebase.json
@@ -0,0 +1,30 @@
+{
+  // Existing firebase configuration ...
+  "database": {
+    // "rules": "database.rules.json"
+  },
+  "firestore": {
+    // "rules": "firestore.rules"
+  },
+
+  // ...
+
+  // Optional emulator configuration. Default
+  // values are used if absent.
+  "emulators": {
+    "firestore": {
+      "host": "0.0.0.0"
+    },
+    // "functions": {
+    //   "port": "5001"
+    // },
+    "database": {
+      // "port": "9000"
+      "host": "0.0.0.0"
+    }
+    // "pubsub": {
+    //   "port": "8085"
+    // }
+
+  }
+}
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
@@ -5,17 +5,17 @@ services:
     build: ./firebase
     stdin_open: true
     tty: true
-    environment:
-     - PYTHONUNBUFFERED=1
-     - CONSUMER_CONFIG_PATH=/code/conf/consumer/consumer.json
-     - KAFKA_CONFIG_PATH=/code/conf/consumer/kafka.json
-     - KAFKA_URL=kafka:29092
-     - AETHER_CONFIG_FIREBASE_PATH=${AETHER_CONFIG_FIREBASE_PATH}
-     - AETHER_FB_CREDENTIAL_PATH=${AETHER_FB_CREDENTIAL_PATH}
-     - AETHER_FB_URL=${AETHER_FB_URL}
-     - AETHER_SERVER_ALIAS=${AETHER_SERVER_ALIAS}
-     - AETHER_FB_HASH_PATH=${AETHER_FB_HASH_PATH}
-     - AETHER_FB_EXPOSE_PORT=${AETHER_FB_EXPOSE_PORT}
+    # environment:
+    #  - PYTHONUNBUFFERED=1
+    #  - CONSUMER_CONFIG_PATH=/code/conf/consumer/consumer.json
+    #  - KAFKA_CONFIG_PATH=/code/conf/consumer/kafka.json
+    #  - KAFKA_URL=kafka:29092
+    #  - AETHER_CONFIG_FIREBASE_PATH=${AETHER_CONFIG_FIREBASE_PATH}
+    #  - AETHER_FB_CREDENTIAL_PATH=${AETHER_FB_CREDENTIAL_PATH}
+    #  - AETHER_FB_URL=${AETHER_FB_URL}
+    #  - AETHER_SERVER_ALIAS=${AETHER_SERVER_ALIAS}
+    #  - AETHER_FB_HASH_PATH=${AETHER_FB_HASH_PATH}
+    #  - AETHER_FB_EXPOSE_PORT=${AETHER_FB_EXPOSE_PORT}
 
     volumes:
      - ./firebase/conf:/code/conf

diff --git a/docker-compose-test.yml b/docker-compose-test.yml
@@ -1,14 +1,48 @@
 version: "2.1"
 services:
 
-  firebases-consumer-test:
+  # -------------------------------------
+  # Firebase Eumulator
+  # -------------------------------------
+
+  emu:
+    image: ehealthafrica/firebase-emulator:latest
+    container_name: firebase_emulator
+    volumes:
+      # Change the Firebase configuration by mounting a local firebase.json
+      - ./conf/test-firebase.json:/app/firebase.json
+    ports:
+      - 5001:5001  # Cloud functions
+      - 9000:9000  # RTDB
+      - 8080:8080  # Cloud Firestore
+      - 8085:8085  # Cloud pub/sub
+      - 5000:5000  # Hosting
+    entrypoint:
+      ["firebase", "emulators:start"]
+
+  consumer-test:
     extends:
       file: docker-compose-base.yml
-      service: firebases-consumer
-    networks:
-      - test_net
-
-networks:
-  test_net:
-    external:
-      name: aether_test
+      service: firebase-consumer
+    environment:
+      # consumer settings
+      CONSUMER_NAME: 'FB-TEST'
+      TENANCY_HEADER: x-oauth-realm
+      # firebase emulator settings
+      FIRESTORE_EMULATOR_HOST: emu:8080
+      FIREBASE_DATABASE_EMULATOR_HOST: emu:9000
+      # redis
+      REDIS_PASSWORD: password
+      # kafka settings
+      KAFKA_CONFIG_PATH: /code/conf/consumer/kafka.json
+      KAFKA_URL: ${KAFKA_URL}
+      SECURITY.PROTOCOL: SASL_SSL
+      SASL.MECHANISM: PLAIN
+      SASL.USERNAME: ${KAFKA_SASL_USERNAME}
+      SASL.PASSWORD: ${KAFKA_SASL_PASSWORD}
+
+  redis:
+    image: redis:alpine
+    command: >
+      redis-server
+      --requirepass password
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,18 +1,34 @@
 version: "2.1"
 services:
 
-  firebase-consumer:
+  consumer:
     extends:
       file: docker-compose-base.yml
       service: firebase-consumer
-    networks:
-      - aether
-    volumes:
-      - ${AETHER_FB_CREDENTIAL_PATH}:/opt/firebase/cert.json
     ports:
-      - "${AETHER_FB_EXPOSE_PORT}:${AETHER_FB_EXPOSE_PORT}"
+      - 9013:9013
+    environment:
+      # consumer settings
+      CONSUMER_NAME: 'FB-TEST'
+      TENANCY_HEADER: x-oauth-realm
+      # redis
+      REDIS_DB: 0
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      REDIS_PASSWORD: password
+      # kafka settings
+      KAFKA_CONFIG_PATH: /code/conf/consumer/kafka.json
+      KAFKA_URL: ${KAFKA_URL}
+      SECURITY.PROTOCOL: SASL_SSL
+      SASL.MECHANISM: PLAIN
+      SASL.USERNAME: ${KAFKA_SASL_USERNAME}
+      SASL.PASSWORD: ${KAFKA_SASL_PASSWORD}
 
-networks:
-  aether:
-    external:
-      name: aether_internal
+  redis:
+    image: redis:alpine
+    command: >
+      redis-server
+      --requirepass password
+      --appendonly yes
+      --auto-aof-rewrite-percentage 100
+      --auto-aof-rewrite-min-size 64mb
diff --git a/firebase/Dockerfile b/firebase/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.6-slim
+FROM python:3.7-slim-stretch
 
 ################################################################################
 ## setup container
@@ -9,18 +9,20 @@ FROM python:3.6-slim
 ## copy files one by one and split commands to use docker cache
 ################################################################################
 
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends gcc  python-dev \
-    && rm -rf /var/lib/apt/lists/*
+RUN apt-get update -qq && \
+    apt-get -qq \
+        --yes \
+        --allow-downgrades \
+        --allow-remove-essential \
+        --allow-change-held-packages \
+        install gcc && \
+    pip install -q --upgrade pip
 
 WORKDIR /code
 
-COPY ./conf/pip /code/conf/pip
-
+COPY ./conf/pip/requirements.txt /code/conf/pip/requirements.txt
 RUN pip3 install -r /code/conf/pip/requirements.txt
 
-RUN apt-get purge -y --auto-remove gcc python-dev
-
 COPY ./ /code
 
 ################################################################################