chore: update release 1.7 (#824)

* chore: transfer deployment to concourse (#811)

* chore(Dockerfile): add LABEL values

* chore: transfer deployment to concourse

* fix: last tweaks

* refactor: rename to deployment

* fix(travis): tests deployment

* fix(kernel): create views within migration phase (#814)

* fix: .dockerignore files (#813)

* chore: upgrade dependencies (#818)

* chore: fix docker and scripts (#820)

* chore: fix ui volumes

* chore: upgrade keycloak 9.0.0

* chore: fix wait_for script

* chore: drop if exists

* fix: wget dry run

* feat(producer): access kernel via REST API (#817)

* fix(producer): reorganize code I

* fix(producer): reorganize code II

* fix: cleaning

* fix: more cleaning

* fix: more cleaning

* fix: more cleaning

* fix(integration): add KAFKA_URL

* fix: drop readonly user

* fix: more cleaning

* feat(producer): access kernel via REST API

* fix: last tweaks

* fix(producer): default server port is 5005

Co-authored-by: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Co-authored-by: obdulia <obdulia.losantos@gmail.com>

.dockerignore

@@ -0,0 +1,25 @@
+**/.dockerignore
+**/Dockerfile
+**/*.Dockerfile
+
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/.Python
+
+**/env
+**/.env
+
+**/.tox
+**/.coverage
+**/.coverage.*
+**/.cache
+**/coverage.xml
+**/*.cover
+
+**/*.log
+
+**/node_modules
+
+!.git/

.gitignore

@@ -82,7 +82,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
-*,cover
+*.cover
 
 # Translations
 *.mo

.travis.yml

@@ -15,38 +15,6 @@ branches:
     - /^[0-9]+(\.[0-9]+){2}$/
 
 
-# define stages and their execution order
-stages:
-  - name: test
-
-  # release only in:
-  #   - branch   develop
-  #   - branch   release-#.#
-  #   - tag      #.#.#
-  #   - never in forks or pull requests
-  - name: release
-    if: |
-      fork IS false AND \
-      type != pull_request AND \
-      ((branch = develop) OR \
-      (branch =~ ^release\-[0-9]+\.[0-9]+$) OR \
-      (tag =~ ^[0-9]+(\.[0-9]+){2}$))
-
-  # Trigger deployment on GCS
-  # only in:
-  #   - branch   develop
-  #   - branch   release-#.#
-  #   - tag      #.#.#
-  #   - never in forks or pull requests
-  - name: deploy
-    if: |
-      fork IS false AND \
-      type != pull_request AND \
-      ((branch = develop) OR \
-      (branch =~ ^release\-[0-9]+\.[0-9]+$) OR \
-      (tag =~ ^[0-9]+(\.[0-9]+){2}$))
-
-
 # use matrix to parallelize tests
 jobs:
   fast_finish: true
@@ -67,13 +35,19 @@ jobs:
       stage: test
       env: 'TEST_MODE=integration'
 
-    - name: "Release"
-      stage: release
-      script: ./scripts/release.sh
+    - name: "Test deployment process"
+      stage: test
+      script: ./scripts/test_deployment.sh
 
-    - name: "Deployment"
+    - name: "Publish images in Docker Hub"
       stage: deploy
-      script: ./scripts/deploy.sh
+      script: ./scripts/release.sh
+      if: |
+        fork IS false AND \
+        type != pull_request AND \
+        ((branch = develop) OR \
+        (branch =~ ^release\-[0-9]+\.[0-9]+$) OR \
+        (tag =~ ^[0-9]+(\.[0-9]+){2}$))
 
 install: true
 

aether-client-library/.dockerignore

@@ -1,19 +1,16 @@
 .dockerignore
 Dockerfile
-db.sqlite3
-__pycache__
-*.pyc
-*.pyo
-*.pyd
-.Python
-env
-pip-log.txt
-pip-delete-this-directory.txt
-.tox
-.coverage
-.coverage.*
-.cache
-coverage.xml
-*,cover
-*.log
-.git
+
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/.Python
+
+**/.coverage
+**/.coverage.*
+**/.cache
+**/coverage.xml
+**/*.cover
+
+**/*.log

aether-kernel/.dockerignore

@@ -1,19 +1,16 @@
 .dockerignore
 Dockerfile
-db.sqlite3
-__pycache__
-*.pyc
-*.pyo
-*.pyd
-.Python
-env
-pip-log.txt
-pip-delete-this-directory.txt
-.tox
-.coverage
-.coverage.*
-.cache
-coverage.xml
-*,cover
-*.log
-.git
+
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/.Python
+
+**/.coverage
+**/.coverage.*
+**/.cache
+**/coverage.xml
+**/*.cover
+
+**/*.log

aether-kernel/Dockerfile

@@ -1,27 +1,31 @@
 FROM python:3.7-slim-buster
 
+LABEL description="Aether Kernel" \
+      name="aether-kernel" \
+      author="eHealth Africa"
+
 ################################################################################
-## setup container
+## set up container
 ################################################################################
 
 COPY ./conf/docker/* /tmp/
 RUN /tmp/setup.sh
 
+WORKDIR /code
+ENTRYPOINT ["/code/entrypoint.sh"]
+
 ################################################################################
-## install app
+## install dependencies
 ## copy files one by one and split commands to use docker cache
 ################################################################################
 
-WORKDIR /code
-
-COPY ./conf/pip /code/conf/pip
+COPY --chown=aether:aether ./conf/pip /code/conf/pip
 RUN pip install -q --upgrade pip && \
     pip install -q -r /code/conf/pip/requirements.txt
-
-COPY ./ /code
+COPY --chown=aether:aether ./ /code
 
 ################################################################################
-## copy application version and create git revision
+## create application version and revision files
 ################################################################################
 
 ARG VERSION=0.0.0
@@ -30,11 +34,3 @@ ARG GIT_REVISION
 RUN mkdir -p /var/tmp && \
     echo $VERSION > /var/tmp/VERSION && \
     echo $GIT_REVISION > /var/tmp/REVISION
-
-################################################################################
-## last setup steps
-################################################################################
-
-RUN chown -R aether: /code
-
-ENTRYPOINT ["/code/entrypoint.sh"]

aether-kernel/aether/kernel/api/serializers.py

@@ -256,6 +256,11 @@ class SchemaDecoratorSerializer(DynamicFieldsMixin, DynamicFieldsModelSerializer
         source='mappings',
     )
 
+    schema_name = serializers.CharField(
+        read_only=True,
+        source='schema.name',
+    )
+
     schema_definition = serializers.JSONField(
         read_only=True,
         source='schema.definition',

aether-kernel/aether/kernel/api/views.py

@@ -45,13 +45,7 @@
 from .exporter import ExporterMixin
 from .mapping_validation import validate_mappings
 
-from . import (
-    filters,
-    models,
-    project_artefacts,
-    serializers,
-    utils
-)
+from . import filters, models, project_artefacts, serializers, utils
 
 
 class ProjectViewSet(MtViewSetMixin, FilteredMixin, viewsets.ModelViewSet):

aether-kernel/aether/kernel/apps.py

@@ -17,8 +17,19 @@
 # under the License.
 
 from django.apps import AppConfig
+from django.db.models.signals import pre_migrate, post_migrate
+
+from aether.kernel.models import pre_migrate_signal, post_migrate_signal
 
 
 class Config(AppConfig):
     name = 'aether.kernel'
     verbose_name = 'Aether Kernel'
+
+    def ready(self):
+        super(Config, self).ready()
+
+        # https://docs.djangoproject.com/en/2.2/ref/signals/#management-signals
+        # enables the migration signals that will recreate the views
+        pre_migrate.connect(pre_migrate_signal, sender=self)
+        post_migrate.connect(post_migrate_signal, sender=self)

aether-kernel/aether/kernel/models.py

@@ -0,0 +1,111 @@
+# Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The purpose of this file is to tell django that there are models linked to
+# this module then it runs the `pre_migrate` and `post_migrate` signals,
+# otherwise they will be ignored.
+
+from django.db import connection
+from django.conf import settings
+
+
+_DROP_VIEWS = '''
+    DROP VIEW IF EXISTS kernel_entity_vw CASCADE;
+    DROP VIEW IF EXISTS kernel_schema_vw CASCADE;
+    DROP VIEW IF EXISTS multitenancy_mtinstance_vw CASCADE;
+'''
+
+_CREATE_MT_VIEW_YES = '''
+    CREATE OR REPLACE VIEW multitenancy_mtinstance_vw AS
+        SELECT realm, instance_id
+        FROM multitenancy_mtinstance
+    ;
+'''
+
+_CREATE_MT_VIEW_NOT = f'''
+    CREATE OR REPLACE VIEW multitenancy_mtinstance_vw AS
+        SELECT '{settings.NO_MULTITENANCY_REALM}' AS realm, id AS instance_id
+        FROM kernel_project
+    ;
+'''
+
+_CREATE_SCHEMA_VIEW = '''
+    CREATE OR REPLACE VIEW kernel_schema_vw AS
+        SELECT
+            GREATEST(sd.modified, s.modified) AS modified,
+
+            sd.id                             AS schemadecorator_id,
+            sd.name                           AS schemadecorator_name,
+
+            s.id                              AS schema_id,
+            s.name                            AS schema_name,
+            s.definition                      AS schema_definition,
+            s.revision                        AS schema_revision,
+
+            mt.realm                          AS realm,
+
+            (s.family = sd.project_id::text)  AS is_identity
+
+        FROM kernel_schemadecorator           AS sd
+        INNER JOIN kernel_schema              AS s
+                ON sd.schema_id = s.id
+        INNER JOIN multitenancy_mtinstance_vw AS mt
+                ON sd.project_id = mt.instance_id
+        ORDER BY 1 ASC
+    ;
+'''
+
+_CREATE_ENTITY_VIEW = '''
+    CREATE OR REPLACE VIEW kernel_entity_vw AS
+        SELECT
+            e.id,
+            e.revision,
+            e.payload,
+            e.modified,
+            e.status,
+
+            s.schemadecorator_id,
+            s.schemadecorator_name,
+            s.schema_name,
+            s.schema_id,
+            s.schema_revision,
+            s.realm
+
+        FROM kernel_entity          AS e
+        INNER JOIN kernel_schema_vw AS s
+                ON e.schemadecorator_id = s.schemadecorator_id
+        ORDER BY e.modified ASC
+    ;
+'''
+
+
+def pre_migrate_signal(**kwargs):
+    with connection.cursor() as cursor:
+        cursor.execute(_DROP_VIEWS)
+
+
+def post_migrate_signal(**kwargs):
+    if settings.MULTITENANCY:
+        create_sql = _CREATE_MT_VIEW_YES
+    else:
+        create_sql = _CREATE_MT_VIEW_NOT
+    create_sql += _CREATE_SCHEMA_VIEW
+    create_sql += _CREATE_ENTITY_VIEW
+
+    with connection.cursor() as cursor:
+        cursor.execute(create_sql)