From 9488826246c3aac3087215b0265852fa705904f5 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Thu, 30 Apr 2020 13:06:48 +0200
Subject: [PATCH 01/29] chore: postgresql-client-11 (buster default) (#832)

---
 aether-kernel/conf/docker/setup.sh     | 2 +-
 aether-odk-module/conf/docker/setup.sh | 2 +-
 aether-ui/conf/docker/setup.sh         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/aether-kernel/conf/docker/setup.sh b/aether-kernel/conf/docker/setup.sh
index 2dc0f1549..61f567bc7 100755
--- a/aether-kernel/conf/docker/setup.sh
+++ b/aether-kernel/conf/docker/setup.sh
@@ -25,7 +25,7 @@ set -Eeuo pipefail
 # define variables
 ################################################################################
 
-POSTGRES_PACKAGE=postgresql-client-10
+POSTGRES_PACKAGE=postgresql-client-11
 
 
 ################################################################################
diff --git a/aether-odk-module/conf/docker/setup.sh b/aether-odk-module/conf/docker/setup.sh
index 2dc0f1549..61f567bc7 100755
--- a/aether-odk-module/conf/docker/setup.sh
+++ b/aether-odk-module/conf/docker/setup.sh
@@ -25,7 +25,7 @@ set -Eeuo pipefail
 # define variables
 ################################################################################
 
-POSTGRES_PACKAGE=postgresql-client-10
+POSTGRES_PACKAGE=postgresql-client-11
 
 
 ################################################################################
diff --git a/aether-ui/conf/docker/setup.sh b/aether-ui/conf/docker/setup.sh
index 2dc0f1549..61f567bc7 100755
--- a/aether-ui/conf/docker/setup.sh
+++ b/aether-ui/conf/docker/setup.sh
@@ -25,7 +25,7 @@ set -Eeuo pipefail
 # define variables
 ################################################################################
 
-POSTGRES_PACKAGE=postgresql-client-10
+POSTGRES_PACKAGE=postgresql-client-11
 
 
 ################################################################################

From 56d0da0a09ee0abb4d91cc711892fb162a7ff6e0 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Thu, 30 Apr 2020 18:41:03 +0200
Subject: [PATCH 02/29] fix(producer): get entities by schema id (#833)

---
 aether-producer/producer/__init__.py   | 30 ++++++------
 aether-producer/producer/kernel.py     |  6 +--
 aether-producer/producer/kernel_api.py | 18 +++----
 aether-producer/producer/kernel_db.py  | 65 ++++++++++++++++----------
 aether-producer/producer/topic.py      | 14 +++---
 aether-producer/tests/__init__.py      |  6 +--
 scripts/test_container.sh              |  3 +-
 7 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/aether-producer/producer/__init__.py b/aether-producer/producer/__init__.py
index 0ad6f26b3..88c18b216 100644
--- a/aether-producer/producer/__init__.py
+++ b/aether-producer/producer/__init__.py
@@ -117,27 +117,25 @@ def broker_info(self):
             md = self.kafka_admin_client.list_topics(timeout=10)
             for b in iter(md.brokers.values()):
                 if b.id == md.controller_id:
-                    res['brokers'].append('{}  (controller)'.format(b))
+                    res['brokers'].append(f'{b}  (controller)')
                 else:
-                    res['brokers'].append('{}'.format(b))
+                    res['brokers'].append(f'{b}')
 
             for t in iter(md.topics.values()):
                 t_str = []
-                if t.error is not None:
-                    errstr = ': {}'.format(t.error)
-                else:
-                    errstr = ''
-
-                t_str.append('{} with {} partition(s){}'.format(t, len(t.partitions), errstr))
+                t_str.append(
+                    f'{t} with {len(t.partitions)} partition(s)'
+                    (f', error: {t.error}' if t.error is not None else '')
+                )
 
                 for p in iter(t.partitions.values()):
-                    if p.error is not None:
-                        errstr = ': {}'.format(p.error)
-                    else:
-                        errstr = ''
-
-                    t_str.append('partition {} leader: {}, replicas: {}, isrs: {}'.format(
-                        p.id, p.leader, p.replicas, p.isrs, errstr))
+                    t_str.append(
+                        f'partition {p.id}'
+                        f', leader: {p.leader}'
+                        f', replicas: {p.replicas}'
+                        f', isrs: {p.isrs}'
+                        (f', error: {p.error}' if p.error is not None else '')
+                    )
                 res['topics'].append(t_str)
             return res
         except Exception as err:
@@ -179,7 +177,7 @@ def check_schemas(self):
                         self.logger.debug(f'Schema {schema_name} unchanged')
 
             # Time between checks for schema change
-            self.safe_sleep(SETTINGS.get('sleep_time', 1))
+            self.safe_sleep(SETTINGS.get('sleep_time', 10))
         self.logger.debug('No longer checking schemas')
 
     # Flask Functions
diff --git a/aether-producer/producer/kernel.py b/aether-producer/producer/kernel.py
index e72621694..6984bc6c1 100644
--- a/aether-producer/producer/kernel.py
+++ b/aether-producer/producer/kernel.py
@@ -64,11 +64,11 @@ def mode(self):
     def get_schemas(self):
         raise NotImplementedError
 
-    def check_updates(self, modified, schema_name, realm):
+    def check_updates(self, realm, schema_id, schema_name, modified):
         raise NotImplementedError
 
-    def count_updates(self, schema_name, realm):
+    def count_updates(self, realm, schema_id, schema_name, modified=''):
         raise NotImplementedError
 
-    def get_updates(self, modified, schema_name, realm):
+    def get_updates(self, realm, schema_id, schema_name, modified):
         raise NotImplementedError
diff --git a/aether-producer/producer/kernel_api.py b/aether-producer/producer/kernel_api.py
index 7ce457ae9..7a29ffd4d 100644
--- a/aether-producer/producer/kernel_api.py
+++ b/aether-producer/producer/kernel_api.py
@@ -42,7 +42,7 @@
     f'{_KERNEL_URL}/'
     'schemadecorators.json?'
     '&page_size={page_size}'
-    '&fields=id,schema_name,schema_definition'
+    '&fields=id,schema,schema_name,schema_definition'
 )
 _ENTITIES_URL = (
     f'{_KERNEL_URL}/'
@@ -74,17 +74,17 @@ def get_schemas(self):
                     _next_url = response['next']
 
                     for entry in response['results']:
-                        yield {'realm': realm, **entry}
+                        yield {'realm': realm, 'schema_id': entry['schema'], **entry}
 
         except Exception:
             self.last_check_error = 'Could not access kernel API to get topics'
             logger.critical(self.last_check_error)
             return []
 
-    def check_updates(self, modified, schema_name, realm):
+    def check_updates(self, realm, schema_id, schema_name, modified):
         url = _ENTITIES_URL.format(
             page_size=1,
-            schema=schema_name,
+            schema=schema_id,
             modified=modified,
         )
         try:
@@ -94,11 +94,11 @@ def check_updates(self, modified, schema_name, realm):
             logger.critical('Could not access kernel API to look for updates')
             return False
 
-    def count_updates(self, schema_name, realm):
+    def count_updates(self, realm, schema_id, schema_name, modified=''):
         url = _ENTITIES_URL.format(
             page_size=1,
-            schema=schema_name,
-            modified='',
+            schema=schema_id,
+            modified=modified or '',
         )
         try:
             _count = self._fetch(url=url, realm=realm)['count']
@@ -108,10 +108,10 @@ def count_updates(self, schema_name, realm):
             logger.critical('Could not access kernel API to look for updates')
             return -1
 
-    def get_updates(self, modified, schema_name, realm):
+    def get_updates(self, realm, schema_id, schema_name, modified):
         url = _ENTITIES_URL.format(
             page_size=self.limit,
-            schema=schema_name,
+            schema=schema_id,
             modified=modified,
         )
 
diff --git a/aether-producer/producer/kernel_db.py b/aether-producer/producer/kernel_db.py
index e9269cd0f..eb96d218f 100644
--- a/aether-producer/producer/kernel_db.py
+++ b/aether-producer/producer/kernel_db.py
@@ -33,32 +33,42 @@
 from producer.kernel import KernelClient, logger
 
 
-_SCHEMAS_SQL = 'SELECT * FROM kernel_schema_vw'
+_SCHEMAS_SQL = '''
+    SELECT schema_id, schema_name, schema_definition, realm
+      FROM kernel_schema_vw
+'''
 
 _CHECK_UPDATES_SQL = '''
     SELECT id
-    FROM kernel_entity_vw
-    WHERE modified    > {modified}
-        AND schema_name = {schema_name}
-        AND realm       = {realm}
-    LIMIT 1;
+      FROM kernel_entity_vw
+     WHERE modified  > {modified}
+       AND schema_id = {schema}
+       AND realm     = {realm}
+     LIMIT 1;
 '''
 
 _COUNT_UPDATES_SQL = '''
     SELECT COUNT(id)
-    FROM kernel_entity_vw
-    WHERE schema_name = {schema_name}
-        AND realm       = {realm};
+      FROM kernel_entity_vw
+     WHERE schema_id = {schema}
+       AND realm     = {realm};
+'''
+_COUNT_MODIFIED_UPDATES_SQL = '''
+    SELECT COUNT(id)
+      FROM kernel_entity_vw
+     WHERE modified  > {modified}
+       AND schema_id = {schema}
+       AND realm     = {realm};
 '''
 
 _GET_UPDATES_SQL = '''
     SELECT *
-    FROM kernel_entity_vw
-    WHERE modified    > {modified}
-        AND schema_name = {schema_name}
-        AND realm       = {realm}
-    ORDER BY modified ASC
-    LIMIT {limit};
+      FROM kernel_entity_vw
+     WHERE modified  > {modified}
+       AND schema_id = {schema}
+       AND realm     = {realm}
+     ORDER BY modified ASC
+     LIMIT {limit};
 '''
 
 
@@ -89,10 +99,10 @@ def get_schemas(self):
             logger.critical('Could not access kernel database to get topics')
             return []
 
-    def check_updates(self, modified, schema_name, realm):
+    def check_updates(self, realm, schema_id, schema_name, modified):
         query = sql.SQL(_CHECK_UPDATES_SQL).format(
             modified=sql.Literal(modified),
-            schema_name=sql.Literal(schema_name),
+            schema=sql.Literal(schema_id),
             realm=sql.Literal(realm),
         )
         cursor = self._exec_sql(schema_name, 1, query)
@@ -102,11 +112,18 @@ def check_updates(self, modified, schema_name, realm):
             logger.critical('Could not access kernel database to look for updates')
             return False
 
-    def count_updates(self, schema_name, realm):
-        query = sql.SQL(_COUNT_UPDATES_SQL).format(
-            schema_name=sql.Literal(schema_name),
-            realm=sql.Literal(realm),
-        )
+    def count_updates(self, realm, schema_id, schema_name, modified=''):
+        if modified:
+            query = sql.SQL(_COUNT_MODIFIED_UPDATES_SQL).format(
+                modified=sql.Literal(modified),
+                schema=sql.Literal(schema_id),
+                realm=sql.Literal(realm),
+            )
+        else:
+            query = sql.SQL(_COUNT_UPDATES_SQL).format(
+                schema=sql.Literal(schema_id),
+                realm=sql.Literal(realm),
+            )
         cursor = self._exec_sql(schema_name, 0, query)
         if cursor:
             _count = cursor.fetchone()[0]
@@ -116,10 +133,10 @@ def count_updates(self, schema_name, realm):
             logger.critical('Could not access kernel database to look for updates')
             return -1
 
-    def get_updates(self, modified, schema_name, realm):
+    def get_updates(self, realm, schema_id, schema_name, modified):
         query = sql.SQL(_GET_UPDATES_SQL).format(
             modified=sql.Literal(modified),
-            schema_name=sql.Literal(schema_name),
+            schema=sql.Literal(schema_id),
             realm=sql.Literal(realm),
             limit=sql.Literal(self.limit),
         )
diff --git a/aether-producer/producer/topic.py b/aether-producer/producer/topic.py
index cd930c9c3..a553941b3 100644
--- a/aether-producer/producer/topic.py
+++ b/aether-producer/producer/topic.py
@@ -61,6 +61,7 @@ class TopicManager(object):
 
     def __init__(self, context, schema, realm):
         self.context = context
+        self.pk = schema['schema_id']
         self.name = schema['schema_name']
         self.realm = realm
         self.offset = ''
@@ -72,7 +73,7 @@ def __init__(self, context, schema, realm):
         self.change_set = {}
         self.successful_changes = []
         self.failed_changes = {}
-        self.sleep_time = int(SETTINGS.get('sleep_time', 2))
+        self.sleep_time = int(SETTINGS.get('sleep_time', 10))
         self.window_size_sec = int(SETTINGS.get('window_size_sec', 3))
 
         self.kafka_failure_wait_time = float(SETTINGS.get('kafka_failure_wait_time', 10))
@@ -172,8 +173,9 @@ def handle_rebuild(self):
         # greened background task to handle rebuilding of topic
         self.operating_status = TopicStatus.REBUILDING
         tag = f'REBUILDING {self.name}:'
-        logger.info(f'{tag} waiting {self.sleep_time *1.5}(sec) for inflight ops to resolve')
-        self.context.safe_sleep(self.sleep_time * 1.5)
+        sleep_time = self.sleep_time * 1.5
+        logger.info(f'{tag} waiting {sleep_time}(sec) for inflight ops to resolve')
+        self.context.safe_sleep(sleep_time)
         logger.info(f'{tag} Deleting Topic')
         self.producer = None
 
@@ -203,13 +205,13 @@ def delete_this_topic(self):
         return False
 
     def updates_available(self):
-        return self.context.kernel_client.check_updates(self.offset, self.name, self.realm)
+        return self.context.kernel_client.check_updates(self.realm, self.pk, self.name, self.offset)
 
     def get_db_updates(self):
-        return self.context.kernel_client.get_updates(self.offset, self.name, self.realm)
+        return self.context.kernel_client.get_updates(self.realm, self.pk, self.name, self.offset)
 
     def get_topic_size(self):
-        return self.context.kernel_client.count_updates(self.name, self.realm)
+        return self.context.kernel_client.count_updates(self.realm, self.pk, self.name)
 
     def update_schema(self, schema_obj):
         self.schema_obj = self.parse_schema(schema_obj)
diff --git a/aether-producer/tests/__init__.py b/aether-producer/tests/__init__.py
index 5a8109728..c68163be9 100644
--- a/aether-producer/tests/__init__.py
+++ b/aether-producer/tests/__init__.py
@@ -40,13 +40,13 @@ def mode(self):
     def get_schemas(self):
         return []
 
-    def check_updates(self, modified, schema_name, realm):
+    def check_updates(self, realm, schema_id, schema_name, modified):
         return False
 
-    def count_updates(self, schema_name, realm):
+    def count_updates(self, realm, schema_id, schema_name, modified=''):
         return 0
 
-    def get_updates(self, modified, schema_name, realm):
+    def get_updates(self, realm, schema_id, schema_name, modified):
         return []
 
 
diff --git a/scripts/test_container.sh b/scripts/test_container.sh
index 14cad63ed..7a36458e6 100755
--- a/scripts/test_container.sh
+++ b/scripts/test_container.sh
@@ -124,7 +124,8 @@ if [[ $1 != "kernel" ]]; then
 
     if [[ $1 = "integration" ]]; then
         build_container producer
-        echo_message "Starting producer"
+        export KERNEL_ACCESS_TYPE=${KERNEL_ACCESS_TYPE:-api}
+        echo_message "Starting producer in mode [${KERNEL_ACCESS_TYPE}]"
         $DC_TEST up -d producer-test
         echo_message "producer ready!"
     fi

From df2b6dd1c751c1d2783b687758f18ab05c1996e3 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Tue, 19 May 2020 15:14:10 +0200
Subject: [PATCH 03/29] feat(odk): upgrade pyxform@1.1.0 (#835)

* feat(odk): upgrade pyxform@1.1.0

* fix: E402 module level import not at top of file
---
 .../aether/client/__init__.py                 |  11 ++-
 .../api/tests/files/demo-xform-multilang.xml  |  20 +++-
 .../aether/odk/api/tests/files/demo-xform.xls | Bin 17920 -> 17920 bytes
 .../aether/odk/api/tests/files/demo-xform.xml |  92 ++++++++++++++----
 .../aether/odk/api/xform_utils.py             |  31 ++++--
 .../conf/pip/primary-requirements.txt         |   3 +-
 aether-odk-module/conf/pip/requirements.txt   |   2 +-
 7 files changed, 119 insertions(+), 40 deletions(-)

diff --git a/aether-client-library/aether/client/__init__.py b/aether-client-library/aether/client/__init__.py
index 51b34d014..c205a1af3 100644
--- a/aether-client-library/aether/client/__init__.py
+++ b/aether-client-library/aether/client/__init__.py
@@ -20,12 +20,7 @@
 from oauthlib import oauth2
 from time import sleep
 from urllib.parse import urlparse
-
-# monkey patch so that bulk insertion works
-from .patches import patched__marshal_object, patched__unmarshal_object
 import bravado_core
-bravado_core.marshal._marshal_object = patched__marshal_object        # noqa
-bravado_core.unmarshal._unmarshal_object = patched__unmarshal_object  # noqa
 
 import bravado
 from bravado.client import (
@@ -42,6 +37,12 @@
 from .oidc import OauthClient
 from .logger import LOG
 
+# monkey patch so that bulk insertion works
+from .patches import patched__marshal_object, patched__unmarshal_object
+
+bravado_core.marshal._marshal_object = patched__marshal_object        # noqa
+bravado_core.unmarshal._unmarshal_object = patched__unmarshal_object  # noqa
+
 _SPEC_URL = '{}/v1/schema/?format=openapi'
 
 
diff --git a/aether-odk-module/aether/odk/api/tests/files/demo-xform-multilang.xml b/aether-odk-module/aether/odk/api/tests/files/demo-xform-multilang.xml
index a5d79b945..31da16833 100644
--- a/aether-odk-module/aether/odk/api/tests/files/demo-xform-multilang.xml
+++ b/aether-odk-module/aether/odk/api/tests/files/demo-xform-multilang.xml
@@ -8,7 +8,7 @@
     xmlns:xsd="http://www.w3.org/2001/XMLSchema">
   <h:head>
     <h:title>My Test Form (multilang)</h:title>
-    <model>
+    <model odk:xforms-version="1.0.0">
       <itext>
         <translation default="true()" lang="English">
           <text id="/Something_that_is_not_None/starttime:label">
@@ -358,12 +358,22 @@
             <index/>
             <value/>
           </iterate>
+          <iterate>
+            <index/>
+            <value/>
+          </iterate>
           <iterate_one jr:template="">
             <it.em/>
           </iterate_one>
+          <iterate_one>
+            <it.em/>
+          </iterate_one>
           <iterate_none jr:template="">
             <nothing/>
           </iterate_none>
+          <iterate_none>
+            <nothing/>
+          </iterate_none>
           <value/>
           <dup_group>
             <value/>
@@ -460,8 +470,8 @@
             nodeset="/Something_that_is_not_None/deviceid"
             required="true()"
             type="string"/>
-      <bind aet="{'emit_level': '0', 'default_visualization': 'bar'}" nodeset="/Something_that_is_not_None/country" required="false()" type="select1"/>
-      <bind aet="{'emit_level': '1'}" nodeset="/Something_that_is_not_None/region" required="false()" type="select1"/>
+      <bind aet="{'emit_level': '0', 'default_visualization': 'bar'}" nodeset="/Something_that_is_not_None/country" required="false()" type="string"/>
+      <bind aet="{'emit_level': '1'}" nodeset="/Something_that_is_not_None/region" required="false()" type="string"/>
       <bind aet="{'emit_level': '2'}" nodeset="/Something_that_is_not_None/city" required="false()" type="string"/>
       <bind nodeset="/Something_that_is_not_None/name" required="false()" type="string"/>
       <bind aet="{'default_visualization': 'line'}" nodeset="/Something_that_is_not_None/location" required="false()" type="geopoint"/>
@@ -473,11 +483,11 @@
       <bind aet="{'emit_level': '4'}" nodeset="/Something_that_is_not_None/number2" required="false()" type="decimal"/>
       <bind nodeset="/Something_that_is_not_None/date" required="false()" type="date"/>
       <bind nodeset="/Something_that_is_not_None/datetime" required="false()" type="dateTime"/>
-      <bind aet="{'emit_level': '3'}" jr:requiredMsg="Choice" nodeset="/Something_that_is_not_None/option" required="true()" type="select1"/>
+      <bind aet="{'emit_level': '3'}" jr:requiredMsg="Choice" nodeset="/Something_that_is_not_None/option" required="true()" type="string"/>
       <bind nodeset="/Something_that_is_not_None/option-a" relevant=" /Something_that_is_not_None/option ='a'"/>
       <bind aet="{'default_visualization': 'pie'}" nodeset="/Something_that_is_not_None/option-a/choice.a" type="string"/>
       <bind nodeset="/Something_that_is_not_None/option-b/choice.b" type="string"/>
-      <bind nodeset="/Something_that_is_not_None/lang" type="select"/>
+      <bind nodeset="/Something_that_is_not_None/lang" type="string"/>
       <bind nodeset="/Something_that_is_not_None/ranking_lang" type="odk:rank"/>
       <bind nodeset="/Something_that_is_not_None/iterate/index" type="int"/>
       <bind nodeset="/Something_that_is_not_None/iterate/value" type="string"/>
diff --git a/aether-odk-module/aether/odk/api/tests/files/demo-xform.xls b/aether-odk-module/aether/odk/api/tests/files/demo-xform.xls
index a96498dd970a1fba9194be6cbecea67e2c51ab66..37290e06baf3d7c724ed20907c1b97344814a224 100644
GIT binary patch
delta 1656
zcmY+EOH5Ni6o${-cM5`FNWfZ2UEJ<KD#qHYf+nsE8;lD#CJ01G5R?jvs4Z$06`!Lb
z;sZ9u#EtPrx0txojooOBi5p$HH9;^T@to=0xwmc7<eNG3&Fk+JW>{f{y?$Hoer*8!
zSdbeV9q*qgLM<$%sKtiffsw)9{#+3nVU~ZdkF#I=Z~eU8=&WG(qCdCclUY{-Fw}Fj
ze<+*XKYVOxFgM`$HF+9a^cUh=*xpi;3mO)mhicIA+wFvGfB|bYA)8=AYF)?{t|`U!
zL~C++W7ryl7_8KVTm`_o{ey`eb;(jJY^KRdddLQrmHbyNdB%?OAKRO|7_Q2LVf<Y~
zBg3x;BZ@Sn6Ow5CVWlQ2PpUqu9ObiHllh!%GepHHMN~0{#54<ctRvBkwEJ*I6Z<^~
z45l!QOM{6g=)q}eFqOfqFj)Btg(Vbcq{0%438}D@f)?aKuo=6;U<<=pX|RQ1QW|V!
zu$6(oYirqgPL|V&dU)&Q(~XJ2os#gzurH{r5|tNtBHhaJyel1N1wNRLCulo#PBo%`
zN)esKMZOrRs1+5LWJRr*>$3kX{h1}#B8i|Yei>m63rqZCdo%0izuW(aSZvD|rG~0q
zxS~iyx~j-P3d3R|1=mesaUrl!SX`Kv<7h?~t|^LIGz>plf?ews8eM3v%YnMk+)xzN
z=t2`D2SXoO4CT^=89B}nmYa&o7M#BvPFhW%p>T$KDu?ry97y4G;kF{u1bL!Av<G1Z
zqn2_$@5phKPZ#b2<i+$9%TQxgXegxNV#*=CCp8pO7r3IR#TJ$z&Q68J!ISaeKDMQ3
z_m0p!kQ%3AF-}?Iw`V3<Pbu<xQD91u*NZMY6?vz`gVYdxkPL`ESaev$&0)Vq4;Eja
zJ&=DW7lIA28)g-0um>I~(vjvA8Ay*6nMm`J=-q3BCyE%-Q$-qdKtYj?^lXzt1M#^c
z6FOmmFK1WZ>^h`ZX8;jDOGl;;^PY}o=V#N0&VR`-j87RE|9wYBXD|HKeYXA|ulf2k

delta 1664
zcmY+EJxmlq6vyA}cL2E@N+O;}&SJp^<U~bRMNO=X9VAvpf+T99C<qD&3o76TevYV!
zAC%e{Wn*WIl@>N88!KuX8%-<(jluWkW@q-egyc8#<~MJ4|2t#TY;2mnep}^#tOopZ
zps&AopnGrvGVrRNMqKDR-gBa>yKe&4!X#g;%CVULu9|aQZwaU8quJG;?Wn&dFxY(b
zCbyC8i)(x^aB2i9z{Fp>r`ZAvRw^~yU_-+y%?@_Zz}M6^<(Jm!(K0B5KUvM?0IWSa
zR=Yi$kIVEhZ5ErFEi8+pca0%q2LG|O?hwPS0yu^L|DloLFM!jE45S`O)W5DYMCBPZ
zN0n#!i_Q74SB_btqE8W3oI_%UgC{nTXh%A|-qFVS01|^O4E@qz;}Z;EKpJdiuuBY1
zSfX%*;=EKiLUBPV9HpR!0!R+ymKa=N7?cKA7>1<5RR&iX_^0N^9mBGmo@f9U710Uy
zO2RjWlvI|B$`RgJXkw%MNFm2UK2*qcjLPatF+Zk=OyfxJ@flW%HUeyco@fR~14!0o
z#JWpzT}HHjIr?7s!tyJroY0GZnPLMAW4_o@$J+VdmR}+)Tf!?+L)AV^C^C>H6<J6+
zLN-zo0*a6iiG?EM!<1Y{JNj@{QQBhQCxI4vwRA$WL}LogH8n??>x$AEQ)rSr;irI=
zNi&pIA8yEXmS|>LQPG0?7b8pEBpQmW))XUqQ!b>)`fy7TX_A!DXC8n~X4+Cr?QOY^
zQtQJVfKY5tVbvO^L_^Wl%@m`1S86D_KHO83wz$HQgxo8!c=%)j;5b%Kr}u>BzSMXn
z&3Hvkbf9&Zb;hYXjRF&=?lgMY4XN8pTvOHQYs!N3HAW9>a)X|C=o;hsod?4i*$586
zZg`-`fIaX~k%{z3k%cs?$VPfBiEhJYm{Y`%o+vV)4c|d>Q<_Lm*DJITpDD8OGk?zA
xV3{AavJ1O9%!+LwzI@1j+jN+Bw$*u^Ckxr1Q(qY$w_2kQZLKDoiI(@f=6~H1^uz!F

diff --git a/aether-odk-module/aether/odk/api/tests/files/demo-xform.xml b/aether-odk-module/aether/odk/api/tests/files/demo-xform.xml
index 02355388b..fae287c21 100644
--- a/aether-odk-module/aether/odk/api/tests/files/demo-xform.xml
+++ b/aether-odk-module/aether/odk/api/tests/files/demo-xform.xml
@@ -9,9 +9,51 @@
         xmlns:xsd="http://www.w3.org/2001/XMLSchema">
   <h:head>
     <h:title>My Test Form</h:title>
-    <model>
+    <model odk:xforms-version="1.0.0">
       <itext>
         <translation default="true()" lang="English">
+          <text id="/Something_that_is_not_None/country/CM:label">
+            <value>Cameroon</value>
+          </text>
+          <text id="/Something_that_is_not_None/country/NG:label">
+            <value>Nigeria</value>
+          </text>
+          <text id="/Something_that_is_not_None/region/CM-1:label">
+            <value>Zone 1</value>
+          </text>
+          <text id="/Something_that_is_not_None/region/CM-2:label">
+            <value>Zone 2</value>
+          </text>
+          <text id="/Something_that_is_not_None/region/NG-N:label">
+            <value>North</value>
+          </text>
+          <text id="/Something_that_is_not_None/region/NG-S:label">
+            <value>South</value>
+          </text>
+          <text id="/Something_that_is_not_None/option/a:label">
+            <value>Option A</value>
+          </text>
+          <text id="/Something_that_is_not_None/option/b:label">
+            <value>Option B</value>
+          </text>
+          <text id="/Something_that_is_not_None/lang/EN:label">
+            <value>English</value>
+          </text>
+          <text id="/Something_that_is_not_None/lang/FR:label">
+            <value>French</value>
+          </text>
+          <text id="/Something_that_is_not_None/lang/GE:label">
+            <value>German</value>
+          </text>
+          <text id="/Something_that_is_not_None/ranking_lang/EN:label">
+            <value>English</value>
+          </text>
+          <text id="/Something_that_is_not_None/ranking_lang/FR:label">
+            <value>French</value>
+          </text>
+          <text id="/Something_that_is_not_None/ranking_lang/GE:label">
+            <value>German</value>
+          </text>
           <text id="static_instance-countries-0">
             <value>Cameroon</value>
           </text>
@@ -78,12 +120,22 @@
             <index/>
             <value/>
           </iterate>
+          <iterate>
+            <index/>
+            <value/>
+          </iterate>
           <iterate_one jr:template="">
             <it.em/>
           </iterate_one>
+          <iterate_one>
+            <it.em/>
+          </iterate_one>
           <iterate_none jr:template="">
             <nothing/>
           </iterate_none>
+          <iterate_none>
+            <nothing/>
+          </iterate_none>
           <value/>
           <dup_group>
             <value/>
@@ -111,23 +163,23 @@
         <root>
           <item>
             <itextId>static_instance-regions-0</itextId>
-            <name>CM-1</name>
             <country>CM</country>
+            <name>CM-1</name>
           </item>
           <item>
             <itextId>static_instance-regions-1</itextId>
-            <name>CM-2</name>
             <country>CM</country>
+            <name>CM-2</name>
           </item>
           <item>
             <itextId>static_instance-regions-2</itextId>
-            <name>NG-N</name>
             <country>NG</country>
+            <name>NG-N</name>
           </item>
           <item>
             <itextId>static_instance-regions-3</itextId>
-            <name>NG-S</name>
             <country>NG</country>
+            <name>NG-S</name>
           </item>
         </root>
       </instance>
@@ -180,8 +232,8 @@
               nodeset="/Something_that_is_not_None/deviceid"
               required="true()"
               type="string"/>
-      <bind aet="{'emit_level': '0', 'default_visualization': 'bar'}" nodeset="/Something_that_is_not_None/country" required="false()" type="select1"/>
-      <bind aet="{'emit_level': '1'}" nodeset="/Something_that_is_not_None/region" required="false()" type="select1"/>
+      <bind aet="{'emit_level': '0', 'default_visualization': 'bar'}" nodeset="/Something_that_is_not_None/country" required="false()" type="string"/>
+      <bind aet="{'emit_level': '1'}" nodeset="/Something_that_is_not_None/region" required="false()" type="string"/>
       <bind aet="{'emit_level': '2'}" nodeset="/Something_that_is_not_None/city" required="false()" type="string"/>
       <bind nodeset="/Something_that_is_not_None/name" required="false()" type="string"/>
       <bind aet="{'default_visualization': 'line'}" nodeset="/Something_that_is_not_None/location" required="false()" type="geopoint"/>
@@ -193,12 +245,12 @@
       <bind aet="{'emit_level': '4'}" nodeset="/Something_that_is_not_None/number2" required="false()" type="decimal"/>
       <bind nodeset="/Something_that_is_not_None/date" required="false()" type="date"/>
       <bind nodeset="/Something_that_is_not_None/datetime" required="false()" type="dateTime"/>
-      <bind aet="{'emit_level': '3'}" jr:requiredMsg="Choice" nodeset="/Something_that_is_not_None/option" required="true()" type="select1"/>
+      <bind aet="{'emit_level': '3'}" jr:requiredMsg="Choice" nodeset="/Something_that_is_not_None/option" required="true()" type="string"/>
       <bind nodeset="/Something_that_is_not_None/option-a" relevant=" /Something_that_is_not_None/option ='a'"/>
       <bind aet="{'default_visualization': 'pie'}" nodeset="/Something_that_is_not_None/option-a/choice.a" type="string"/>
       <bind nodeset="/Something_that_is_not_None/option-b" relevant=" /Something_that_is_not_None/option ='b'"/>
       <bind nodeset="/Something_that_is_not_None/option-b/choice.b" type="string"/>
-      <bind nodeset="/Something_that_is_not_None/lang" type="select"/>
+      <bind nodeset="/Something_that_is_not_None/lang" type="string"/>
       <bind nodeset="/Something_that_is_not_None/ranking_lang" type="odk:rank"/>
       <bind nodeset="/Something_that_is_not_None/iterate/index" type="int"/>
       <bind nodeset="/Something_that_is_not_None/iterate/value" type="string"/>
@@ -222,11 +274,11 @@
     <select1 ref="/Something_that_is_not_None/country">
       <label>Country</label>
       <item>
-        <label>Cameroon</label>
+        <label ref="jr:itext('/Something_that_is_not_None/country/CM:label')"/>
         <value>CM</value>
       </item>
       <item>
-        <label>Nigeria</label>
+        <label ref="jr:itext('/Something_that_is_not_None/country/NG:label')"/>
         <value>NG</value>
       </item>
     </select1>
@@ -273,11 +325,11 @@
     <select1 ref="/Something_that_is_not_None/option">
       <label>Choice (A/B)</label>
       <item>
-        <label>Option A</label>
+        <label ref="jr:itext('/Something_that_is_not_None/option/a:label')"/>
         <value>a</value>
       </item>
       <item>
-        <label>Option B</label>
+        <label ref="jr:itext('/Something_that_is_not_None/option/b:label')"/>
         <value>b</value>
       </item>
     </select1>
@@ -296,37 +348,37 @@
     <select ref="/Something_that_is_not_None/lang">
       <label>Spoken languages</label>
       <item>
-        <label>English</label>
+        <label ref="jr:itext('/Something_that_is_not_None/lang/EN:label')"/>
         <value>EN</value>
       </item>
       <item>
-        <label>French</label>
+        <label ref="jr:itext('/Something_that_is_not_None/lang/FR:label')"/>
         <value>FR</value>
       </item>
       <item>
-        <label>German</label>
+        <label ref="jr:itext('/Something_that_is_not_None/lang/GE:label')"/>
         <value>GE</value>
       </item>
     </select>
     <odk:rank ref="/Something_that_is_not_None/ranking_lang">
       <label>Ranking of languages</label>
       <item>
-        <label>English</label>
+        <label ref="jr:itext('/Something_that_is_not_None/ranking_lang/EN:label')"/>
         <value>EN</value>
       </item>
       <item>
-        <label>French</label>
+        <label ref="jr:itext('/Something_that_is_not_None/ranking_lang/FR:label')"/>
         <value>FR</value>
       </item>
       <item>
-        <label>German</label>
+        <label ref="jr:itext('/Something_that_is_not_None/ranking_lang/GE:label')"/>
         <value>GE</value>
       </item>
     </odk:rank>
     <group ref="/Something_that_is_not_None/iterate">
       <label>Indicate loop elements</label>
       <repeat
-                nodeset="/Something_that_is_not_None/iterate">
+              nodeset="/Something_that_is_not_None/iterate">
         <input ref="/Something_that_is_not_None/iterate/index">
           <label>Index</label>
         </input>
diff --git a/aether-odk-module/aether/odk/api/xform_utils.py b/aether-odk-module/aether/odk/api/xform_utils.py
index c43c690b1..d6f7e0985 100644
--- a/aether-odk-module/aether/odk/api/xform_utils.py
+++ b/aether-odk-module/aether/odk/api/xform_utils.py
@@ -686,14 +686,22 @@ def __get_xform_instance_skeleton(xml_definition):
                 schema[xpath]['type'] = bind_entry.get('@type')
                 schema[xpath]['required'] = bind_entry.get('@required') == 'true()'
 
-                if schema[xpath]['type'] in SELECT_TAGS:
-                    select_options = __get_xform_choices(xform_dict, xpath, itexts)
-                    if select_options:
-                        schema[xpath]['choices'] = select_options
             if AET_TAG in bind_entry:
                 xpath = bind_entry.get('@nodeset')
                 schema[xpath]['annotations'] = __parse_annotations(bind_entry.get(AET_TAG))
 
+    # search in body all the SELECT_TAGS entries
+    for tag in SELECT_TAGS:
+        for entries in __find_in_dict(xform_dict, tag):
+            entries = __wrap_as_list(entries)
+            for select_entry in entries:
+                xpath = select_entry.get('@ref')
+                schema[xpath]['type'] = tag
+
+                select_options = __get_xform_choices(xform_dict, xpath, itexts)
+                if select_options:
+                    schema[xpath]['choices'] = select_options
+
     # search in body all the repeat entries
     for entries in __find_in_dict(xform_dict, 'repeat'):
         entries = __wrap_as_list(entries)
@@ -974,19 +982,28 @@ def __get_all_paths(dictionary):
 
     It's only used to get the jsonpaths (or xpaths)
     of the instance skeleton defined in the xForm.
-
-    Assumption: there are no lists in the skeleton.
     '''
     def walk(obj, parent_keys=[]):
         for k, v in obj.items():
             is_dict = isinstance(v, dict)
+            is_list = isinstance(v, list)
             if k.startswith('@'):  # ignore attributes
                 continue
             keys = parent_keys + [k]
             xpath = '/' + '/'.join(keys)
-            paths.append((xpath, isinstance(v, dict)))
+            paths.append((xpath, is_dict or is_list))
             if is_dict:
                 walk(v, keys)
+            elif is_list:
+                # in pyxform 1.x.x there could be duplicated entries like:
+                #    <entry jr:template="">
+                #       <child/>
+                #    </entry>
+                #    <entry>
+                #       <child/>
+                #    </entry>
+                # ignore the first entry
+                walk(v[1], keys)
 
     paths = []
     walk(dictionary)
diff --git a/aether-odk-module/conf/pip/primary-requirements.txt b/aether-odk-module/conf/pip/primary-requirements.txt
index 95b6663ae..310f6ce68 100644
--- a/aether-odk-module/conf/pip/primary-requirements.txt
+++ b/aether-odk-module/conf/pip/primary-requirements.txt
@@ -22,6 +22,5 @@ aether.sdk[cache,server,storage,test]
 # xForm and data manipulation
 lxml
 python-dateutil
-# BREAKING CHANGES
-pyxform<1
+pyxform
 spavro
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index b3bc04f84..61e032808 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -70,7 +70,7 @@ pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
-pyxform==0.15.1
+pyxform==1.1.0
 redis==3.4.1
 requests==2.23.0
 rsa==4.0

From f1cca5370ff33c1e9597c119bd65967d437d155e Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 25 May 2020 11:08:20 +0200
Subject: [PATCH 04/29] fix(uwsgi): static-expires missing in strict mode
 (#838)

* fix(uwsgi): static-expires unavailable in strict mode

* fix: either route
---
 aether-kernel/conf/uwsgi/config.ini     | 2 --
 aether-odk-module/conf/uwsgi/config.ini | 2 --
 aether-ui/conf/uwsgi/config.ini         | 2 --
 3 files changed, 6 deletions(-)

diff --git a/aether-kernel/conf/uwsgi/config.ini b/aether-kernel/conf/uwsgi/config.ini
index a2da15cf1..1025badbd 100644
--- a/aether-kernel/conf/uwsgi/config.ini
+++ b/aether-kernel/conf/uwsgi/config.ini
@@ -50,9 +50,7 @@ http              = 0.0.0.0:$(WEB_SERVER_PORT)
 
 # ------------------------------------------------------------------------------
 if-env            = CUSTOM_UWSGI_SERVE_STATIC
-static-expires    = /* 7776000
 static-map        = $(STATIC_URL)=$(STATIC_ROOT)
-route             = */favicon.ico$ static:$(STATIC_ROOT)/aether/images/aether.ico
 endif             =
 # ------------------------------------------------------------------------------
 
diff --git a/aether-odk-module/conf/uwsgi/config.ini b/aether-odk-module/conf/uwsgi/config.ini
index a2da15cf1..1025badbd 100644
--- a/aether-odk-module/conf/uwsgi/config.ini
+++ b/aether-odk-module/conf/uwsgi/config.ini
@@ -50,9 +50,7 @@ http              = 0.0.0.0:$(WEB_SERVER_PORT)
 
 # ------------------------------------------------------------------------------
 if-env            = CUSTOM_UWSGI_SERVE_STATIC
-static-expires    = /* 7776000
 static-map        = $(STATIC_URL)=$(STATIC_ROOT)
-route             = */favicon.ico$ static:$(STATIC_ROOT)/aether/images/aether.ico
 endif             =
 # ------------------------------------------------------------------------------
 
diff --git a/aether-ui/conf/uwsgi/config.ini b/aether-ui/conf/uwsgi/config.ini
index bc8b5684f..224cbbcf2 100644
--- a/aether-ui/conf/uwsgi/config.ini
+++ b/aether-ui/conf/uwsgi/config.ini
@@ -50,9 +50,7 @@ http              = 0.0.0.0:$(WEB_SERVER_PORT)
 
 # ------------------------------------------------------------------------------
 if-env            = CUSTOM_UWSGI_SERVE_STATIC
-static-expires    = /* 7776000
 static-map        = $(STATIC_URL)=$(STATIC_ROOT)
-route             = */favicon.ico$ static:$(STATIC_ROOT)/aether/images/aether.ico
 endif             =
 # ------------------------------------------------------------------------------
 

From 0db06129f1830e20202bc5d686579b3dd9c7cf51 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Fri, 29 May 2020 11:43:34 +0200
Subject: [PATCH 05/29] refactor(ui): move to React Hooks III (#819)

* refactor: cleaning styles

* chore: keycloak 9.0.0

* refactor: simplify Modal component

* fix: click outside pipeline actions

* fix(docker): ui volumes

* fix(ui): css lint

* fix(ui): small cleaning

* fix(ui): Settings component I

* fix(ui): Settings component II

* fix(ui): Settings component III

* fix(ui): Settings component IV

* fix(ui): Pipeline component I

* feat(ui): Fullscreen component

* fix(ui): Pipeline component II

* fix(ui):  css lint class-name-format

* fix(ui): Pipeline component III

* fix(ui): schema name cannot start with digits

* fix(ui): Pipeline component IV

* chore(ui): use uuid package

* fix: small tweaks

* test(ui): use data-test attribute

* test(ui): Settings component

* fix(ui): remove useless test

* fix: refresh pipeline page and add contract

* fix: sort imports (1st third party)
---
 .../apps/components/AvroSchemaViewer.jsx      |   8 +-
 .../apps/components/AvroSchemaViewer.spec.jsx |  74 +-
 .../{ModalDialog.jsx => Fullscreen.jsx}       |  21 +-
 .../ui/assets/apps/components/Modal.jsx       |  45 +-
 .../ui/assets/apps/components/NavBar.jsx      |   6 +-
 .../ui/assets/apps/components/NavBar.spec.jsx |  14 +-
 .../ui/assets/apps/components/index.jsx       |   2 +
 .../ui/assets/apps/pipeline/Pipeline.jsx      | 669 ++++--------------
 .../ui/assets/apps/pipeline/Pipeline.spec.jsx |  63 --
 .../pipeline/components/ContractAddButton.jsx |  17 +-
 .../apps/pipeline/components/ContractCard.jsx |   9 +-
 .../components/ContractRemoveButton.jsx       |  93 +++
 .../apps/pipeline/components/ContractTabs.jsx |  98 +++
 .../apps/pipeline/components/DeleteModal.jsx  |   3 +-
 .../apps/pipeline/components/DeleteStatus.jsx |   3 +-
 .../pipeline/components/IdentityContract.jsx  | 180 +++++
 .../pipeline/components/PipelineActions.jsx   |  31 +-
 .../apps/pipeline/components/PipelineInfo.jsx |   2 +-
 .../components/PipelineRemoveButton.jsx       |   2 +-
 .../components/PipelineRenameButton.jsx       |   6 +-
 .../pipeline/components/SubmissionCard.jsx    |   2 +-
 .../aether/ui/assets/apps/pipeline/redux.jsx  | 140 ++--
 .../ui/assets/apps/pipeline/redux.spec.jsx    |   8 +-
 .../assets/apps/pipeline/sections/Input.jsx   |   2 +-
 .../apps/pipeline/sections/Sections.jsx       | 159 +++++
 .../apps/pipeline/sections/Settings.jsx       | 465 ++++--------
 .../apps/pipeline/sections/Settings.spec.jsx  | 426 ++++++-----
 .../aether/ui/assets/apps/utils/constants.jsx |   3 +-
 .../aether/ui/assets/apps/utils/index.jsx     |  11 +-
 aether-ui/aether/ui/assets/conf/sass-lint.yml |   4 +-
 .../ui/assets/css/_pipeline-settings.scss     |  16 +-
 aether-ui/aether/ui/assets/css/_pipeline.scss | 120 ++--
 .../aether/ui/assets/css/_pipelines.scss      |  26 +-
 .../ui/assets/css/_section-entity-types.scss  |  22 +-
 .../aether/ui/assets/css/_section-input.scss  |  26 +-
 .../ui/assets/css/_section-mapping.scss       |  19 +-
 .../aether/ui/assets/css/base/_global.scss    |  15 +-
 .../Portal.jsx => css/base/_keyframes.scss}   |  62 +-
 .../aether/ui/assets/css/base/_messages.scss  |   2 +-
 .../aether/ui/assets/css/base/_mixins.scss    |  13 +
 .../aether/ui/assets/css/base/_modal.scss     |   2 -
 .../aether/ui/assets/css/base/_nav-bar.scss   |   7 -
 .../aether/ui/assets/css/base/_settings.scss  |  22 +-
 .../ui/assets/css/base/_styleguide.scss       |   1 -
 aether-ui/aether/ui/assets/css/index.scss     |   3 +
 aether-ui/aether/ui/assets/package.json       |   1 +
 docker-compose-base.yml                       |   1 -
 47 files changed, 1382 insertions(+), 1542 deletions(-)
 rename aether-ui/aether/ui/assets/apps/components/{ModalDialog.jsx => Fullscreen.jsx} (67%)
 delete mode 100644 aether-ui/aether/ui/assets/apps/pipeline/Pipeline.spec.jsx
 create mode 100644 aether-ui/aether/ui/assets/apps/pipeline/components/ContractRemoveButton.jsx
 create mode 100644 aether-ui/aether/ui/assets/apps/pipeline/components/ContractTabs.jsx
 create mode 100644 aether-ui/aether/ui/assets/apps/pipeline/components/IdentityContract.jsx
 create mode 100644 aether-ui/aether/ui/assets/apps/pipeline/sections/Sections.jsx
 rename aether-ui/aether/ui/assets/{apps/components/Portal.jsx => css/base/_keyframes.scss} (50%)

diff --git a/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.jsx b/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.jsx
index 37de47d91..6af012450 100644
--- a/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.jsx
@@ -21,9 +21,9 @@
 import React from 'react'
 import { FormattedMessage, defineMessages, useIntl } from 'react-intl'
 
-import { AVRO_EXTENDED_TYPE, MASKING_ANNOTATION, MASKING_PUBLIC } from '../utils/constants'
 import { clone, isEmpty, generateGUID } from '../utils'
 import { parseSchema, isOptionalType, isPrimitive, typeToString } from '../utils/avro-utils'
+import { AVRO_EXTENDED_TYPE, MASKING_ANNOTATION, MASKING_PUBLIC } from '../utils/constants'
 
 const MESSAGES = defineMessages({
   nullable: {
@@ -107,7 +107,7 @@ const AvroSchemaViewer = ({ className, hideChildren, highlight, pathPrefix, sche
       if (children) {
         children = (
           <div
-            data-qa={jsonPath + '.$'}
+            data-test={jsonPath + '.$'}
             key={generateGUID()}
             className={(isUnion ? 'group-union' : 'group-list')}
           >
@@ -118,7 +118,7 @@ const AvroSchemaViewer = ({ className, hideChildren, highlight, pathPrefix, sche
     }
 
     return (
-      <div data-qa={jsonPath} key={generateGUID()} className='group'>
+      <div data-test={jsonPath} key={generateGUID()} className='group'>
         {
           field.name &&
             <div className={highlightedClassName + (children ? ' group-title' : ' field')}>
@@ -149,7 +149,7 @@ const AvroSchemaViewer = ({ className, hideChildren, highlight, pathPrefix, sche
     return (
       <div className={className}>
         <div className='group'>
-          <div data-qa={`group-title-${schema.name}`} className='title group-title'>
+          <div data-test={`group-title-${schema.name}`} className='title group-title'>
             {schema.name}
           </div>
           <div className='properties group-list'>
diff --git a/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.spec.jsx b/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.spec.jsx
index a9100e21e..2d101dccf 100644
--- a/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.spec.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/AvroSchemaViewer.spec.jsx
@@ -21,8 +21,8 @@
 /* global describe, it, expect */
 
 import React from 'react'
-import { mountWithIntl } from '../../tests/enzyme-helpers'
 
+import { mountWithIntl } from '../../tests/enzyme-helpers'
 import { mockInputSchema } from '../../tests/mock'
 import { AvroSchemaViewer } from '../components'
 
@@ -63,9 +63,9 @@ describe('AvroSchemaViewer', () => {
       />
     )
 
-    expect(component.find('[data-qa="$.id"]').html())
+    expect(component.find('[data-test="$.id"]').html())
       .toContain('<div class="input-schema-mapped-1 field"><span class="name">id</span>')
-    expect(component.find('[data-qa="$.dictionary.code"]').html())
+    expect(component.find('[data-test="$.dictionary.code"]').html())
       .toContain('<div class="input-schema-mapped-2 field"><span class="name">code</span>')
   })
 
@@ -74,132 +74,132 @@ describe('AvroSchemaViewer', () => {
       <AvroSchemaViewer schema={mockInputSchema} />
     )
 
-    expect(component.find('[data-qa^="group-title-"]').length).toEqual(1)
+    expect(component.find('[data-test^="group-title-"]').length).toEqual(1)
 
     // PRIMITIVES
 
-    const idDiv = component.find('[data-qa="id"]').html()
+    const idDiv = component.find('[data-test="id"]').html()
     expect(idDiv).toContain('<div class=" field">')
     expect(idDiv).not.toContain('<i class="fas fa-lock"></i>') // public
     expect(idDiv).toContain('<span class="name">id</span>')
     expect(idDiv).toContain('<span class="type">string</span>')
 
-    const textDiv = component.find('[data-qa="text"]').html()
+    const textDiv = component.find('[data-test="text"]').html()
     expect(textDiv).toContain('<div class=" field">')
     expect(textDiv).toContain('<span class="name">text</span>')
     expect(textDiv).toContain('<span class="type">string (nullable)</span>')
 
-    const choicesDiv = component.find('[data-qa="choices"]').html()
+    const choicesDiv = component.find('[data-test="choices"]').html()
     expect(choicesDiv).toContain('<div class=" field">')
     expect(choicesDiv).toContain('<span class="name">choices</span>')
     expect(choicesDiv).toContain('<span class="type">enum (a, b)</span>')
 
     // RECORDS
 
-    const dictionaryDiv = component.find('[data-qa="dictionary"]').html()
-    expect(dictionaryDiv).toContain('<div data-qa="dictionary" class="group">')
+    const dictionaryDiv = component.find('[data-test="dictionary"]').html()
+    expect(dictionaryDiv).toContain('<div data-test="dictionary" class="group">')
     expect(dictionaryDiv).toContain('<div class=" group-title">')
     expect(dictionaryDiv).toContain('<span class="name">dictionary</span>')
     expect(dictionaryDiv).toContain('<span class="type">record</span>')
 
-    const dictionaryCodeDiv = component.find('[data-qa="dictionary.code"]').html()
+    const dictionaryCodeDiv = component.find('[data-test="dictionary.code"]').html()
     expect(dictionaryCodeDiv).toContain('<div class=" field">')
     expect(dictionaryCodeDiv).toContain('<span class="name">code</span>')
     expect(dictionaryCodeDiv).toContain('<span class="type">int</span>')
 
-    const listNumbersDiv = component.find('[data-qa="list_of_numbers"]').html()
+    const listNumbersDiv = component.find('[data-test="list_of_numbers"]').html()
     expect(listNumbersDiv).toContain('<div class=" field">')
     expect(listNumbersDiv).toContain('<span class="name">list_of_numbers</span>')
     expect(listNumbersDiv).toContain('<span class="type">array [union: int, boolean (nullable)]</span>')
 
     // ARRAYS
 
-    const listTextsDiv = component.find('[data-qa="list_of_texts"]').html()
+    const listTextsDiv = component.find('[data-test="list_of_texts"]').html()
     expect(listTextsDiv).toContain('<div class=" field">')
     expect(listTextsDiv).toContain('<span class="name">list_of_texts</span>')
     expect(listTextsDiv).toContain('<span class="type">array [string]</span>')
 
-    const listDictionariesDiv = component.find('[data-qa="list_of_dictionaries"]').html()
+    const listDictionariesDiv = component.find('[data-test="list_of_dictionaries"]').html()
     expect(listDictionariesDiv).toContain('<div class=" field">')
     expect(listDictionariesDiv).toContain('<span class="name">list_of_dictionaries</span>')
     expect(listDictionariesDiv).toContain('<span class="type">array [record]</span>')
 
-    const listDictionariesWordDiv = component.find('[data-qa="list_of_dictionaries.#.word"]').html()
+    const listDictionariesWordDiv = component.find('[data-test="list_of_dictionaries.#.word"]').html()
     expect(listDictionariesWordDiv).toContain('<div class=" field">')
     expect(listDictionariesWordDiv).toContain('<span class="name">word</span>')
     expect(listDictionariesWordDiv).toContain('<span class="type">string</span>')
 
-    const listDictionariesMeaningDiv = component.find('[data-qa="list_of_dictionaries.#.meaning"]').html()
+    const listDictionariesMeaningDiv = component.find('[data-test="list_of_dictionaries.#.meaning"]').html()
     expect(listDictionariesMeaningDiv).toContain('<div class=" field">')
     expect(listDictionariesMeaningDiv).toContain('<span class="name">meaning</span>')
     expect(listDictionariesMeaningDiv).toContain('<span class="type">string</span>')
 
     // MAPS
 
-    const mapPrimitivesDiv = component.find('[data-qa="mapping_primitives"]').html()
+    const mapPrimitivesDiv = component.find('[data-test="mapping_primitives"]').html()
     expect(mapPrimitivesDiv).toContain('<div class=" field">')
     expect(mapPrimitivesDiv).toContain('<span class="name">mapping_primitives</span>')
     expect(mapPrimitivesDiv).toContain('<span class="type">map {float} (nullable)</span>')
 
-    const mapDictionariesDiv = component.find('[data-qa="mapping_dictionaries"]').html()
+    const mapDictionariesDiv = component.find('[data-test="mapping_dictionaries"]').html()
     expect(mapDictionariesDiv).toContain('<div class=" field">')
     expect(mapDictionariesDiv).toContain('<span class="name">mapping_dictionaries</span>')
     expect(mapDictionariesDiv).toContain('<span class="type">map {record}</span>')
 
-    const mapDictionariesXDiv = component.find('[data-qa="mapping_dictionaries.#.x"]').html()
+    const mapDictionariesXDiv = component.find('[data-test="mapping_dictionaries.#.x"]').html()
     expect(mapDictionariesXDiv).toContain('<div class=" field">')
     expect(mapDictionariesXDiv).toContain('<span class="name">x</span>')
     expect(mapDictionariesXDiv).toContain('<span class="type">double</span>')
 
-    const mapDictionariesYDiv = component.find('[data-qa="mapping_dictionaries.#.y"]').html()
+    const mapDictionariesYDiv = component.find('[data-test="mapping_dictionaries.#.y"]').html()
     expect(mapDictionariesYDiv).toContain('<div class=" field">')
     expect(mapDictionariesYDiv).toContain('<span class="name">y</span>')
     expect(mapDictionariesYDiv).toContain('<span class="type">float</span>')
 
     // UNIONS
 
-    const unionPrimitivesDiv = component.find('[data-qa="primitive_union"]').html()
+    const unionPrimitivesDiv = component.find('[data-test="primitive_union"]').html()
     expect(unionPrimitivesDiv).toContain('<div class=" field">')
     expect(unionPrimitivesDiv).toContain('<span class="name">primitive_union</span>')
     expect(unionPrimitivesDiv).toContain('<span class="type">union: int, string (nullable)</span>')
 
-    const unionComplexDiv = component.find('[data-qa="complex_union"]').html()
+    const unionComplexDiv = component.find('[data-test="complex_union"]').html()
     expect(unionComplexDiv).toContain('<div class=" group-title">')
     expect(unionComplexDiv).toContain('<span class="name">complex_union</span>')
     expect(unionComplexDiv).toContain('<span class="type">union (nullable)</span>')
 
-    const unionComplexBooleanDiv = component.find('[data-qa="complex_union.1"]').html()
+    const unionComplexBooleanDiv = component.find('[data-test="complex_union.1"]').html()
     expect(unionComplexBooleanDiv).toContain('<div class=" field">')
     expect(unionComplexBooleanDiv).toContain('<span class="name">1</span>')
     expect(unionComplexBooleanDiv).toContain('<span class="type">boolean</span>')
 
-    const unionComplexStringDiv = component.find('[data-qa="complex_union.2"]').html()
+    const unionComplexStringDiv = component.find('[data-test="complex_union.2"]').html()
     expect(unionComplexStringDiv).toContain('<div class=" field">')
     expect(unionComplexStringDiv).toContain('<span class="name">2</span>')
     expect(unionComplexStringDiv).toContain('<span class="type">string</span>')
 
-    const unionComplexRecordDiv = component.find('[data-qa="complex_union.3"]').html()
+    const unionComplexRecordDiv = component.find('[data-test="complex_union.3"]').html()
     expect(unionComplexRecordDiv).toContain('<div class=" group-title">')
     expect(unionComplexRecordDiv).toContain('<span class="name">3</span>')
     expect(unionComplexRecordDiv).toContain('<span class="type">record</span>')
 
     // Extended types and non public
 
-    const locationDiv = component.find('[data-qa="location"]').html()
-    expect(locationDiv).toContain('<div data-qa="location" class="group">')
+    const locationDiv = component.find('[data-test="location"]').html()
+    expect(locationDiv).toContain('<div data-test="location" class="group">')
     expect(locationDiv).toContain('<div class=" group-title">')
     expect(locationDiv).toContain('<i class="fas fa-lock"></i>') // restricted
     expect(locationDiv).toContain('<span class="name">location</span>')
     expect(locationDiv).toContain('<span class="type">geopoint (nullable)</span>') // not record
 
-    const timestampDiv = component.find('[data-qa="timestamp"]').html()
-    expect(timestampDiv).toContain('<div data-qa="timestamp" class="group">')
+    const timestampDiv = component.find('[data-test="timestamp"]').html()
+    expect(timestampDiv).toContain('<div data-test="timestamp" class="group">')
     expect(timestampDiv).toContain('<div class=" field">')
     expect(timestampDiv).toContain('<span class="name">timestamp</span>')
     expect(timestampDiv).toContain('<span class="type">dateTime</span>') // not string
 
-    const updatedAtDiv = component.find('[data-qa="updated_at"]').html()
-    expect(updatedAtDiv).toContain('<div data-qa="updated_at" class="group">')
+    const updatedAtDiv = component.find('[data-test="updated_at"]').html()
+    expect(updatedAtDiv).toContain('<div data-test="updated_at" class="group">')
     expect(updatedAtDiv).toContain('<div class=" field">')
     expect(updatedAtDiv).toContain('<span class="name">updated_at</span>')
     expect(updatedAtDiv).toContain('<span class="type">dateTime (nullable)</span>') // not string
@@ -210,26 +210,26 @@ describe('AvroSchemaViewer', () => {
       <AvroSchemaViewer schema={mockInputSchema} hideChildren />
     )
 
-    expect(component.find('[data-qa^="group-title-"]').length).toEqual(1)
+    expect(component.find('[data-test^="group-title-"]').length).toEqual(1)
 
     // 1st level
 
-    const idDiv = component.find('[data-qa="id"]').html()
+    const idDiv = component.find('[data-test="id"]').html()
     expect(idDiv).toContain('<div class=" field">')
     expect(idDiv).toContain('<span class="name">id</span>')
     expect(idDiv).toContain('<span class="type">string</span>')
 
-    const dictionaryDiv = component.find('[data-qa="dictionary"]').html()
-    expect(dictionaryDiv).toContain('<div data-qa="dictionary" class="group">')
+    const dictionaryDiv = component.find('[data-test="dictionary"]').html()
+    expect(dictionaryDiv).toContain('<div data-test="dictionary" class="group">')
     expect(dictionaryDiv).toContain('<div class=" field">')
     expect(dictionaryDiv).toContain('<span class="name">dictionary</span>')
     expect(dictionaryDiv).toContain('<span class="type">record</span>')
 
     // 2nd Level
-    expect(component.find('[data-qa="dictionary.code"]')).toEqual({})
+    expect(component.find('[data-test="dictionary.code"]')).toEqual({})
 
-    const locationDiv = component.find('[data-qa="location"]').html()
-    expect(locationDiv).toContain('<div data-qa="location" class="group">')
+    const locationDiv = component.find('[data-test="location"]').html()
+    expect(locationDiv).toContain('<div data-test="location" class="group">')
     expect(locationDiv).toContain('<div class=" field">')
     expect(locationDiv).toContain('<i class="fas fa-lock"></i>')
     expect(locationDiv).toContain('<span class="name">location</span>')
diff --git a/aether-ui/aether/ui/assets/apps/components/ModalDialog.jsx b/aether-ui/aether/ui/assets/apps/components/Fullscreen.jsx
similarity index 67%
rename from aether-ui/aether/ui/assets/apps/components/ModalDialog.jsx
rename to aether-ui/aether/ui/assets/apps/components/Fullscreen.jsx
index 4ad0879a8..b6d1faeaf 100644
--- a/aether-ui/aether/ui/assets/apps/components/ModalDialog.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/Fullscreen.jsx
@@ -19,19 +19,16 @@
  */
 
 import React from 'react'
+import { FormattedMessage } from 'react-intl'
 
-const ModalDialog = ({ header, children, buttons }) => (
-  <div className='modal-dialog'>
-    <div className='modal-header'>
-      <span className='modal-title'>{header}</span>
-    </div>
-
-    <div className='modal-content'>
-      {children}
-
-      <div className='modal-actions'>{buttons}</div>
-    </div>
+const Fullscreen = ({ value, toggle }) => (
+  <div className='btn-icon fullscreen-toggle' onClick={() => { toggle() }}>
+    {
+      value
+        ? <FormattedMessage id='fullscreen.off' defaultMessage='fullscreen off' />
+        : <FormattedMessage id='fullscreen.on' defaultMessage='fullscreen on' />
+    }
   </div>
 )
 
-export default ModalDialog
+export default Fullscreen
diff --git a/aether-ui/aether/ui/assets/apps/components/Modal.jsx b/aether-ui/aether/ui/assets/apps/components/Modal.jsx
index f7e781f37..40eef0115 100644
--- a/aether-ui/aether/ui/assets/apps/components/Modal.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/Modal.jsx
@@ -18,15 +18,50 @@
  * under the License.
  */
 
-import React from 'react'
+import React, { useState, useEffect } from 'react'
+import ReactDOM from 'react-dom'
 
-import Portal from './Portal'
-import ModalDialog from './ModalDialog'
+// https://reactjs.org/docs/portals.html
 
-const Modal = ({ onEnter, onEscape, ...props }) => (
+const Portal = ({ children, onEscape, onEnter }) => {
+  const [element] = useState(document.createElement('div'))
+
+  useEffect(() => {
+    const onKeyDown = (event) => {
+      if (event.key === 'Escape' && onEscape) {
+        onEscape(event)
+      }
+      if (event.key === 'Enter' && onEnter) {
+        onEnter(event)
+      }
+    }
+
+    document.body.appendChild(element)
+    document.addEventListener('keydown', onKeyDown)
+
+    return () => {
+      document.body.removeChild(element)
+      document.removeEventListener('keydown', onKeyDown)
+    }
+  })
+
+  return ReactDOM.createPortal(children, element)
+}
+
+const Modal = ({ onEnter, onEscape, header, children, buttons }) => (
   <Portal onEnter={onEnter} onEscape={onEscape}>
     <div className='modal show'>
-      <ModalDialog {...props} />
+      <div className='modal-dialog'>
+        <div className='modal-header'>
+          <span className='modal-title'>{header}</span>
+        </div>
+
+        <div className='modal-content'>
+          {children}
+
+          <div className='modal-actions'>{buttons}</div>
+        </div>
+      </div>
     </div>
   </Portal>
 )
diff --git a/aether-ui/aether/ui/assets/apps/components/NavBar.jsx b/aether-ui/aether/ui/assets/apps/components/NavBar.jsx
index 3b622b7bd..2ef1e27d0 100644
--- a/aether-ui/aether/ui/assets/apps/components/NavBar.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/NavBar.jsx
@@ -36,7 +36,7 @@ const NavBar = ({ children, showBreadcrumb, onClick }) => {
   const logoutUrl = window.location.origin + window.location.pathname + 'logout'
 
   return (
-    <div data-qa='navbar' className='navbar top-nav'>
+    <div data-test='navbar' className='navbar top-nav'>
       <a className='top-nav-logo' title='aether' onClick={onClick}>
         <div className='logo-container'>
           <div className='flipper'>
@@ -49,12 +49,12 @@ const NavBar = ({ children, showBreadcrumb, onClick }) => {
 
       {
         showBreadcrumb &&
-          <div data-qa='navbar-breadcrumb' className='top-nav-breadcrumb'>
+          <div data-test='navbar-breadcrumb' className='top-nav-breadcrumb'>
             {children}
           </div>
       }
 
-      <div data-qa='navbar-user' className='top-nav-user'>
+      <div data-test='navbar-user' className='top-nav-user'>
         <span className='user-name'>
           {user.name}
         </span>
diff --git a/aether-ui/aether/ui/assets/apps/components/NavBar.spec.jsx b/aether-ui/aether/ui/assets/apps/components/NavBar.spec.jsx
index 689bc2e26..1b755b759 100644
--- a/aether-ui/aether/ui/assets/apps/components/NavBar.spec.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/NavBar.spec.jsx
@@ -21,9 +21,9 @@
 /* global describe, it, expect, beforeEach */
 
 import React from 'react'
-import { mountWithIntl } from '../../tests/enzyme-helpers'
 import { MemoryRouter } from 'react-router'
 
+import { mountWithIntl } from '../../tests/enzyme-helpers'
 import NavBar from './NavBar'
 
 const mountWithRouter = (node) => mountWithIntl(<MemoryRouter>{node}</MemoryRouter>)
@@ -39,15 +39,15 @@ describe('NavBar', () => {
 
   it('should render the nav bar', () => {
     const component = mountWithRouter(<NavBar />)
-    expect(component.find('[data-qa="navbar"]').exists()).toBeTruthy()
-    expect(component.find('[data-qa="navbar-user"]').exists()).toBeTruthy()
-    expect(component.find('[data-qa="navbar-user"]').html()).toContain('user test')
-    expect(component.find('[data-qa="navbar-breadcrumb"]').exists()).toBeFalsy()
+    expect(component.find('[data-test="navbar"]').exists()).toBeTruthy()
+    expect(component.find('[data-test="navbar-user"]').exists()).toBeTruthy()
+    expect(component.find('[data-test="navbar-user"]').html()).toContain('user test')
+    expect(component.find('[data-test="navbar-breadcrumb"]').exists()).toBeFalsy()
   })
 
   it('should include the breadcrumb', () => {
     const component = mountWithRouter(<NavBar showBreadcrumb />)
-    const breadcrumb = component.find('[data-qa="navbar-breadcrumb"]')
+    const breadcrumb = component.find('[data-test="navbar-breadcrumb"]')
     expect(breadcrumb.exists()).toBeTruthy()
   })
 
@@ -57,7 +57,7 @@ describe('NavBar', () => {
         breadcrumb...
       </NavBar>
     )
-    const breadcrumb = component.find('[data-qa="navbar-breadcrumb"]')
+    const breadcrumb = component.find('[data-test="navbar-breadcrumb"]')
     expect(breadcrumb.exists()).toBeTruthy()
     expect(breadcrumb.html()).toContain('breadcrumb...')
   })
diff --git a/aether-ui/aether/ui/assets/apps/components/index.jsx b/aether-ui/aether/ui/assets/apps/components/index.jsx
index ccf866696..50bbccb09 100644
--- a/aether-ui/aether/ui/assets/apps/components/index.jsx
+++ b/aether-ui/aether/ui/assets/apps/components/index.jsx
@@ -22,6 +22,7 @@
 import AppLayout from './AppLayout'
 import AvroSchemaViewer from './AvroSchemaViewer'
 import Clipboard from './Clipboard'
+import Fullscreen from './Fullscreen'
 import LoadingSpinner from './LoadingSpinner'
 import Modal from './Modal'
 import ModalError from './ModalError'
@@ -33,6 +34,7 @@ export {
   AppLayout,
   AvroSchemaViewer,
   Clipboard,
+  Fullscreen,
   LoadingSpinner,
   Modal,
   ModalError,
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.jsx b/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.jsx
index 4bd49e6d3..5f0bcf6bf 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.jsx
@@ -18,573 +18,204 @@
  * under the License.
  */
 
-import React, { Component } from 'react'
+import React, { useEffect, useState } from 'react'
+import { useParams } from 'react-router'
+import { useHistory } from 'react-router-dom'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
 
-import { LoadingSpinner, Modal, ModalError, NavBar } from '../components'
+import { LoadingSpinner, Modal, NavBar } from '../components'
 
-import Input from './sections/Input'
-import EntityTypes from './sections/EntityTypes'
-import Mapping from './sections/Mapping'
-import Output from './sections/Output'
+import Sections from './sections/Sections'
 import Settings from './sections/Settings'
-import DeleteModal from './components/DeleteModal'
-import DeleteStatus from './components/DeleteStatus'
+import ContractTabs from './components/ContractTabs'
 
-import {
+import { clearSelection, getPipelineById, selectContract } from './redux'
+import { PIPELINE_SECTION_INPUT } from '../utils/constants'
+
+const Pipeline = ({
+  pipeline,
+  contract,
+  section,
+  newContract,
   clearSelection,
   getPipelineById,
-  selectContract,
-  selectSection,
-  deleteContract
-} from './redux'
-
-import {
-  PIPELINE_SECTION_INPUT,
-  CONTRACT_SECTION_ENTITY_TYPES,
-  CONTRACT_SECTION_MAPPING,
-  CONTRACT_SECTION_SETTINGS
-} from '../utils/constants'
-
-class Pipeline extends Component {
-  constructor (props) {
-    super(props)
-
-    const view = props.section || props.match.params.section || PIPELINE_SECTION_INPUT
-
-    this.state = {
-      deleteOptions: {},
-      fullscreen: false,
-      isNew: props.location.state && props.location.state.isNewContract,
-      newContract: null,
-      showCancelModal: false,
-      showDeleteModal: false,
-      showDeleteProgress: false,
-      showSettings: (view === CONTRACT_SECTION_SETTINGS),
-      showOutput: false,
-      onContractSavedCallback: null,
-      view: (view === CONTRACT_SECTION_SETTINGS) ? CONTRACT_SECTION_ENTITY_TYPES : view
-    }
-
-    this.handleAddNewContract = this.handleAddNewContract.bind(this)
-    this.deleteContract = this.deleteContract.bind(this)
-    this.hideModalProgress = this.hideModalProgress.bind(this)
-    this.handleBackToPipelines = this.handleBackToPipelines.bind(this)
-    this.handleCancelContract = this.handleCancelContract.bind(this)
-    this.handleContracts = this.handleContracts.bind(this)
-    this.onContractTabSelected = this.onContractTabSelected.bind(this)
-    this.handleDeleteContract = this.handleDeleteContract.bind(this)
-    this.handleInput = this.handleInput.bind(this)
-    this.handleCreateNewContract = this.handleCreateNewContract.bind(this)
-    this.handleSave = this.handleSave.bind(this)
-    this.handleCloseSettings = this.handleCloseSettings.bind(this)
-  }
-
-  componentDidMount () {
-    // load current pipeline using location address (router props)
-    if (!this.state.isNew) {
-      this.props.getPipelineById(this.props.match.params.pid)
+  selectContract
+}) => {
+  const { pid, cid, view } = useParams()
+  const history = useHistory()
+
+  const [fullscreen, setFullscreen] = useState(false)
+  const [showOutput, setShowOutput] = useState(false)
+  const [showSettings, setShowSettings] = useState(!!newContract)
+  const [showUnsavedWarning, setShowUnsavedWarning] = useState(false)
+  const [unsavedCallback, setUnsavedCallback] = useState(null)
+
+  useEffect(() => {
+    if (!pipeline || pipeline.id !== pid) {
+      getPipelineById(pid)
     } else {
-      this.handleAddNewContract()
-    }
-  }
-
-  componentDidUpdate (prevProps) {
-    if (
-      !this.state.isNew &&
-      this.props.contract &&
-      (
-        prevProps.section !== this.props.section ||
-        (prevProps.contract && prevProps.contract.id !== this.props.contract.id)
-      )
-    ) {
-      // update router history
-      this.props.history.push(
-        `/${this.props.pipeline.id}/${this.props.contract.id}/${this.props.section}`
-      )
-    }
-
-    // persist in-memory new contract
-    if (this.state.isNew && this.props.contract && this.props.contract !== this.state.newContract) {
-      this.setState({
-        newContract: this.props.contract
-      })
-    }
-
-    if (prevProps.section !== this.props.section) {
-      // update state
-      if (this.props.section === PIPELINE_SECTION_INPUT) {
-        return this.setState({
-          view: this.props.section,
-          showSettings: false,
-          showOutput: false,
-          fullscreen: false
-        })
+      if (!newContract && contract && (section !== view || contract.id !== cid)) {
+        // update router history
+        history.push(`/${pipeline.id}/${contract.id}/${section}`)
       }
 
-      if (this.props.section === CONTRACT_SECTION_SETTINGS) {
-        return this.setState({
-          view: this.state.view === PIPELINE_SECTION_INPUT
-            ? CONTRACT_SECTION_ENTITY_TYPES
-            : this.state.view,
-          showSettings: true,
-          showOutput: false
-        })
+      if (section === PIPELINE_SECTION_INPUT) {
+        setShowSettings(false)
+        setShowOutput(false)
       }
 
-      return this.setState({
-        view: this.props.section,
-        showSettings: false
-      })
+      if (newContract) {
+        setShowSettings(true)
+      }
     }
-  }
+  })
 
-  handleBackToPipelines () {
-    this.checkUnsavedContract(() => {
-      this.props.history.push('/')
-      this.props.clearSelection()
-    })
+  if (!pipeline) {
+    return <LoadingSpinner /> // still loading data
   }
 
-  checkUnsavedContract (cb) {
-    if (this.state.newContract) {
-      this.setState({
-        showCancelModal: true,
-        onContractSavedCallback: cb
-      })
+  const checkUnsavedContract = (callback) => {
+    if (newContract) {
+      setShowUnsavedWarning(true)
+      setUnsavedCallback(callback)
     } else {
-      this.setState({
-        showCancelModal: false
-      }, () => { cb && cb() })
+      setShowUnsavedWarning(false)
+      setUnsavedCallback(null)
+      callback && callback()
     }
   }
 
-  render () {
-    const { pipeline } = this.props
-    if (!pipeline) {
-      return <LoadingSpinner /> // still loading data
-    }
-
-    const fullscreenDiv = (
-      <div
-        className='btn-icon fullscreen-toggle'
-        onClick={() => { this.setState({ fullscreen: !this.state.fullscreen }) }}
-      >
-        {
-          this.state.fullscreen
-            ? <FormattedMessage id='pipeline.navbar.shrink' defaultMessage='fullscreen off' />
-            : <FormattedMessage id='pipeline.navbar.fullscreen' defaultMessage='fullscreen on' />
-        }
-      </div>
-    )
-
-    return (
-      <div className={`pipelines-container show-pipeline pipeline--${this.state.view}`}>
-        {this.props.loading && <LoadingSpinner />}
-        {this.props.error && <ModalError error={this.props.error} />}
-        <NavBar showBreadcrumb onClick={this.handleBackToPipelines}>
-          <div className='breadcrumb-links'>
-            <a onClick={this.handleBackToPipelines}>
-              <FormattedMessage
-                id='pipeline.navbar.pipelines'
-                defaultMessage='Pipelines'
-              />
-            </a>
-            <span className='breadcrumb-pipeline-name'>
-              <span>// </span>
-              {pipeline.name}
-              {
-                pipeline.isInputReadOnly &&
-                  <span className='tag'>
-                    <FormattedMessage
-                      id='pipeline.navbar.read-only'
-                      defaultMessage='read-only'
-                    />
-                  </span>
-              }
-            </span>
-          </div>
-        </NavBar>
-
-        <div className={`
-          pipeline
-          ${this.state.showOutput ? 'show-output' : ''}
-          ${this.state.fullscreen ? 'fullscreen' : ''}
-        `}
-        >
-          <div className='pipeline-tabs'>
-            {this.renderContractTabs()}
-            {this.renderNewContractTab()}
-            {
-              !this.state.newContract &&
-                <button
-                  type='button'
-                  className='btn btn-c btn-sm new-contract'
-                  onClick={this.handleAddNewContract}
-                >
-                  <span className='details-title'>
-                    <FormattedMessage
-                      id='contract.add.button'
-                      defaultMessage='Add contract'
-                    />
-                  </span>
-                </button>
-            }
-          </div>
-
-          {this.renderSectionTabs()}
-
-          {
-            this.state.showSettings &&
-              <Settings
-                onClose={this.handleCloseSettings}
-                isNew={this.state.isNew}
-                onDelete={this.handleDeleteContract}
-                onSave={this.handleSave}
-                onNew={this.handleCreateNewContract}
-              />
-          }
-          <div className='pipeline-sections'>
-            <div className='pipeline-section__input'><Input /></div>
-            {
-              this.props.contract &&
-                <div className='pipeline-section__entityTypes'>
-                  <EntityTypes />
-                  {fullscreenDiv}
-                </div>
-            }
-            {
-              this.props.contract &&
-                <div className='pipeline-section__mapping'>
-                  <Mapping />
-                  {fullscreenDiv}
-                </div>
-            }
-          </div>
-          {this.props.contract && <div className='pipeline-output'><Output /></div>}
-        </div>
-        {this.renderCancelationModal()}
-        {this.renderDeletionModal()}
-        {this.renderDeleteProgressModal()}
-      </div>
-    )
-  }
-
-  handleCreateNewContract (contract) {
-    this.setState({
-      newContract: contract
-    })
-  }
-
-  handleAddNewContract () {
-    this.setState({
-      isNew: true
-    }, () => { this.props.selectSection(CONTRACT_SECTION_SETTINGS) })
-  }
-
-  handleCloseSettings () {
-    if (this.state.isNew) {
-      this.checkUnsavedContract(null)
-    } else {
-      this.props.selectSection(
-        this.state.view === CONTRACT_SECTION_SETTINGS
-          ? CONTRACT_SECTION_ENTITY_TYPES
-          : this.state.view
-      )
-    }
-  }
-
-  handleDeleteContract () {
-    this.setState({
-      showDeleteModal: true
-    })
-  }
-
-  deleteContract (deleteOptions) {
-    this.props.deleteContract(this.props.contract.id, deleteOptions)
-    this.setState({
-      deleteOptions,
-      showDeleteModal: false,
-      showDeleteProgress: true
+  const backToList = () => {
+    checkUnsavedContract(() => {
+      history.push('/')
+      clearSelection()
     })
   }
 
-  renderCancelationModal () {
-    if (!this.state.showCancelModal) {
-      return null
-    }
-
-    const header = (
-      <FormattedMessage
-        id='pipeline.new.contract.header'
-        defaultMessage='Cancel the new contract?'
-      />
-    )
-    const close = () => { this.setState({ showCancelModal: false }) }
-    const buttons = (
-      <div>
-        <button
-          data-qa='pipeline.new.contract.continue'
-          className='btn btn-w'
-          onClick={close}
-        >
-          <FormattedMessage
-            id='pipeline.new.contract.continue.message'
-            defaultMessage='No, Continue Editing'
-          />
-        </button>
-
-        <button className='btn btn-w btn-primary' onClick={this.handleCancelContract}>
-          <FormattedMessage
-            id='pipeline.new.contract.cancel'
-            defaultMessage='Yes, Cancel'
-          />
-        </button>
-      </div>
-    )
-
-    return (
-      <Modal
-        header={header}
-        buttons={buttons}
-        onEscape={close}
-        onEnter={this.handleCancelContract}
-      />
-    )
-  }
-
-  renderDeletionModal () {
-    if (!this.state.showDeleteModal) {
-      return null
-    }
+  const closeUnsavedWarning = () => { setShowUnsavedWarning(false) }
+  const cancelUnsavedWarning = () => {
+    setShowUnsavedWarning(false)
+    setShowSettings(false)
 
-    return (
-      <DeleteModal
-        onClose={() => { this.setState({ showDeleteModal: false }) }}
-        onDelete={(options) => { this.deleteContract(options) }}
-        objectType='contract'
-        obj={this.props.contract}
-      />
-    )
-  }
+    const nextContractId = contract
+      ? contract.id
+      : pipeline.contracts.length > 0
+        ? pipeline.contracts[0].id
+        : null
 
-  renderDeleteProgressModal () {
-    if (!this.state.showDeleteProgress) {
-      return null
-    }
-    return (
-      <DeleteStatus
-        header={
-          <FormattedMessage
-            id='contract.delete.status.header'
-            defaultMessage='Deleting contract '
-          />
-        }
-        deleteOptions={this.state.deleteOptions}
-        toggle={this.hideModalProgress}
-        showModal={this.state.showDeleteProgress}
-      />
+    selectContract(
+      pipeline.id,
+      nextContractId,
+      nextContractId ? section : PIPELINE_SECTION_INPUT
     )
-  }
-
-  hideModalProgress () {
-    this.setState({ showDeleteProgress: false })
-  }
-
-  handleCancelContract () {
-    this.setState({
-      newContract: null,
-      isNew: false,
-      showCancelModal: false
-    })
-    const nextContract = this.props.pipeline.contracts.length > 0
-      ? this.props.pipeline.contracts[0]
-      : null
-
-    this.props.selectContract(nextContract)
-    this.props.selectSection(nextContract ? this.state.view : PIPELINE_SECTION_INPUT)
-
-    if (this.state.onContractSavedCallback) {
-      this.state.onContractSavedCallback()
-      this.setState({
-        onContractSavedCallback: null
-      })
-    }
-  }
-
-  handleSave (view) {
-    this.setState({
-      newContract: null,
-      isNew: false
-    }, () => {
-      this.props.selectSection(view || CONTRACT_SECTION_ENTITY_TYPES)
-    })
-  }
 
-  onContractTabSelected (contract) {
-    this.checkUnsavedContract(() => { this.props.selectContract(contract.pipeline, contract.id) })
+    unsavedCallback && unsavedCallback()
+    setUnsavedCallback(null)
   }
 
-  toggleSettings () {
-    if (this.state.showSettings) {
-      this.props.selectSection(CONTRACT_SECTION_ENTITY_TYPES)
-    } else if (!this.state.showSettings) {
-      this.props.selectSection(CONTRACT_SECTION_SETTINGS)
-    }
-  }
-
-  renderContractTabs () {
-    return this.props.pipeline.contracts.map(contract => (
-      <div
-        key={contract.id}
-        className={`pipeline-tab ${contract.id === this.props.contract.id ? 'active' : ''}`}
-        onClick={() => { this.onContractTabSelected(contract) }}
+  const unsavedWarningButtons = (
+    <>
+      <button
+        data-test='pipeline.new.contract.continue'
+        className='btn btn-w'
+        onClick={closeUnsavedWarning}
       >
-        <span className='contract-name'>{contract.name}</span>
-
-        {
-          (contract.mapping_errors || []).length > 0 &&
-            <span className={`status ${(contract.mapping_errors || []).length ? 'red' : 'green'}`} />
-        }
-
-        <div
-          className={`btn-icon settings-button ${this.state.showSettings ? 'active' : ''}`}
-          onClick={() => { this.toggleSettings() }}
-        >
-          <i className='fas fa-wrench' />
+        <FormattedMessage
+          id='pipeline.new.contract.continue.message'
+          defaultMessage='No, Continue Editing'
+        />
+      </button>
+
+      <button className='btn btn-w btn-primary' onClick={cancelUnsavedWarning}>
+        <FormattedMessage
+          id='pipeline.new.contract.cancel'
+          defaultMessage='Yes, Cancel'
+        />
+      </button>
+    </>
+  )
+
+  return (
+    <div className={`pipelines-container show-pipeline pipeline--${section}`}>
+      <NavBar showBreadcrumb onClick={backToList}>
+        <div className='breadcrumb-links'>
+          <a onClick={backToList}>
+            <FormattedMessage
+              id='pipeline.navbar.pipelines'
+              defaultMessage='Pipelines'
+            />
+          </a>
+          <span className='breadcrumb-pipeline-name'>
+            <span>// </span>
+            {pipeline.name}
+            {
+              pipeline.isInputReadOnly &&
+                <span className='tag'>
+                  <FormattedMessage
+                    id='pipeline.navbar.read-only'
+                    defaultMessage='read-only'
+                  />
+                </span>
+            }
+          </span>
         </div>
-      </div>
-    ))
-  }
+      </NavBar>
 
-  renderNewContractTab () {
-    const newContract = this.state.newContract
-    return newContract && !newContract.created && (
       <div
-        key={newContract.id}
-        className={`pipeline-tab ${newContract.id === this.props.contract.id ? 'active' : ''}`}
+        className={`pipeline ${showOutput ? 'show-output' : ''} ${fullscreen ? 'fullscreen' : ''}`}
       >
-        <span className='contract-name new'>
-          <FormattedMessage
-            id='pipeline.tab.newContract'
-            defaultMessage='new Contract'
-          />
-        </span>
-      </div>
-    )
-  }
-
-  handleContracts () {
-    if (!this.props.pipeline.contracts.length) {
-      this.handleAddNewContract()
-    } else {
-      this.props.selectSection(CONTRACT_SECTION_ENTITY_TYPES)
-    }
-  }
-
-  handleInput () {
-    this.checkUnsavedContract(() => {
-      this.props.selectSection(PIPELINE_SECTION_INPUT)
-    })
-  }
-
-  renderSectionTabs () {
-    const { contract = {} } = this.props
-
-    return (
-      <div className='pipeline-nav'>
-        <div className='pipeline-nav-items'>
-          <div
-            className='pipeline-nav-item__input'
-            onClick={this.handleInput}
-          >
-            <div className='badge'>
-              <i className='fas fa-file' />
-            </div>
-            <FormattedMessage
-              id='pipeline.navbar.input'
-              defaultMessage='Input'
-            />
-          </div>
-
-          <div
-            className='pipeline-nav-item__entityTypes'
-            onClick={() => { this.props.selectSection(CONTRACT_SECTION_ENTITY_TYPES) }}
-          >
-            <div className='badge'>
-              <i className='fas fa-caret-right' />
-            </div>
-            <FormattedMessage
-              id='pipeline.navbar.entity.types'
-              defaultMessage='Entity Types'
-            />
-
-          </div>
+        <ContractTabs
+          checkUnsavedContract={checkUnsavedContract}
+          showSettings={showSettings}
+          toggleSettings={() => { setShowSettings(!showSettings) }}
+        />
+
+        <Sections
+          checkUnsavedContract={checkUnsavedContract}
+          fullscreen={fullscreen}
+          toggleFullscreen={() => { setFullscreen(!fullscreen) }}
+          toggleOutput={() => { setShowOutput(!showOutput) }}
+        />
 
-          <div
-            className='pipeline-nav-item__mapping'
-            onClick={() => { this.props.selectSection(CONTRACT_SECTION_MAPPING) }}
-          >
-            <div className='badge'>
-              <i className='fas fa-caret-right' />
-            </div>
-            <FormattedMessage
-              id='pipeline.navbar.mapping'
-              defaultMessage='Mapping'
+        {
+          (newContract || showSettings) &&
+            <Settings
+              onClose={() => { checkUnsavedContract(() => { setShowSettings(false) }) }}
+              onSave={() => { setShowSettings(false) }}
             />
+        }
 
-          </div>
-
-          <div
-            className='pipeline-nav-item__contracts'
-            onClick={this.handleContracts}
-          >
-            <div className='badge'>
-              <i className='fas fa-caret-right' />
-            </div>
-            <FormattedMessage
-              id='pipeline.navbar.contracts'
-              defaultMessage='Contracts'
+        {
+          showUnsavedWarning &&
+            <Modal
+              header={
+                <FormattedMessage
+                  id='pipeline.unsaved.contract.warning.header'
+                  defaultMessage='Cancel the new contract?'
+                />
+              }
+              buttons={unsavedWarningButtons}
+              onEscape={closeUnsavedWarning}
+              onEnter={cancelUnsavedWarning}
             />
-          </div>
-        </div>
-
-        <div
-          className='pipeline-nav-item__output'
-          onClick={() => { this.setState({ showOutput: !this.state.showOutput }) }}
-        >
-          <div className='badge'>
-            <i className='fas fa-caret-right' />
-          </div>
-          <FormattedMessage
-            id='pipeline.navbar.output'
-            defaultMessage='Output'
-          />
-          {
-            ((contract && contract.mapping_errors) || []).length > 0 &&
-              <span className={`status ${(contract.mapping_errors || []).length ? 'red' : 'green'}`} />
-          }
-        </div>
+        }
       </div>
-    )
-  }
+    </div>
+  )
 }
 
 const mapStateToProps = ({ pipelines }) => ({
-  loading: pipelines.loading,
-  section: pipelines.currentSection,
-  pipeline: pipelines.currentPipeline,
   contract: pipelines.currentContract,
-  error: pipelines.error,
-  deleteStatus: pipelines.deleteStatus
+  newContract: pipelines.newContract,
+  pipeline: pipelines.currentPipeline,
+  section: pipelines.currentSection || PIPELINE_SECTION_INPUT
 })
+
 const mapDispatchToProps = {
   clearSelection,
   getPipelineById,
-  selectContract,
-  selectSection,
-  deleteContract
+  selectContract
 }
 
 export default connect(mapStateToProps, mapDispatchToProps)(Pipeline)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.spec.jsx b/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.spec.jsx
deleted file mode 100644
index c0a547ab3..000000000
--- a/aether-ui/aether/ui/assets/apps/pipeline/Pipeline.spec.jsx
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
- *
- * See the NOTICE file distributed with this work for additional information
- * regarding copyright ownership.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/* global describe, it, expect, beforeEach */
-
-import React from 'react'
-import { createStore, applyMiddleware } from 'redux'
-import { Provider } from 'react-redux'
-import { mountWithIntl } from '../../tests/enzyme-helpers'
-
-import Pipeline from '../pipeline/Pipeline'
-import reducer from '../redux/reducers'
-import middleware from '../redux/middleware'
-
-import {
-  CONTRACT_SECTION_SETTINGS
-} from '../utils/constants'
-
-describe('Pipeline Component', () => {
-  let store
-
-  beforeEach(() => {
-    // create a new store instance for each test
-    store = createStore(
-      reducer,
-      applyMiddleware(...middleware)
-    )
-  })
-
-  it('should initiate a new contract process', () => {
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Pipeline
-          location={{ state: { isNewContract: true } }}
-          match={{ params: {} }}
-        />
-      </Provider>
-    )
-
-    const reduxState = store.getState().pipelines
-    const localState = component.find('Pipeline').state()
-
-    expect(reduxState.currentSection).toEqual(CONTRACT_SECTION_SETTINGS)
-    expect(localState.isNew).toEqual(true)
-  })
-})
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractAddButton.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractAddButton.jsx
index 052805bd8..03941227a 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractAddButton.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractAddButton.jsx
@@ -23,23 +23,20 @@ import { useHistory } from 'react-router-dom'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
 
-import { selectPipeline } from '../redux'
+import { startNewContract } from '../redux'
 
-const ContractAddButton = ({ className, pipeline: { id }, selectPipeline }) => {
+const ContractAddButton = ({ className, pipeline: { id }, startNewContract }) => {
   const history = useHistory()
-  const handleCreateNewContract = () => {
-    selectPipeline(id)
-    history.push({
-      pathname: `/${id}`,
-      state: { isNewContract: true }
-    })
+  const startAddNewContract = () => {
+    startNewContract(id)
+    history.push(`/${id}`)
   }
 
   return (
     <button
       type='button'
       className={className || 'btn btn-c'}
-      onClick={handleCreateNewContract}
+      onClick={startAddNewContract}
     >
       <span className='details-title'>
         <FormattedMessage
@@ -52,6 +49,6 @@ const ContractAddButton = ({ className, pipeline: { id }, selectPipeline }) => {
 }
 
 const mapStateToProps = () => ({})
-const mapDispatchToProps = { selectPipeline }
+const mapDispatchToProps = { startNewContract }
 
 export default connect(mapStateToProps, mapDispatchToProps)(ContractAddButton)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractCard.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractCard.jsx
index e090ef488..d45c33ef7 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractCard.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractCard.jsx
@@ -25,14 +25,13 @@ import { FormattedMessage } from 'react-intl'
 
 import ContractPublishButton from './ContractPublishButton'
 
-import { selectContract, selectSection } from '../redux'
+import { selectContract } from '../redux'
 import { CONTRACT_SECTION_ENTITY_TYPES } from '../../utils/constants'
 
-const ContractCard = ({ contract, selectContract, selectSection }) => {
+const ContractCard = ({ contract, selectContract }) => {
   const history = useHistory()
   const handleSelectContract = (pipelineId, contractId) => {
-    selectContract(pipelineId, contractId)
-    selectSection(CONTRACT_SECTION_ENTITY_TYPES)
+    selectContract(pipelineId, contractId, CONTRACT_SECTION_ENTITY_TYPES)
     history.push(`/${pipelineId}/${contractId}/${CONTRACT_SECTION_ENTITY_TYPES}`)
   }
 
@@ -94,6 +93,6 @@ const ContractCard = ({ contract, selectContract, selectSection }) => {
 }
 
 const mapStateToProps = () => ({})
-const mapDispatchToProps = { selectContract, selectSection }
+const mapDispatchToProps = { selectContract }
 
 export default connect(mapStateToProps, mapDispatchToProps)(ContractCard)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractRemoveButton.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractRemoveButton.jsx
new file mode 100644
index 000000000..d97684f50
--- /dev/null
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractRemoveButton.jsx
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+ *
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useState } from 'react'
+import { connect } from 'react-redux'
+import { FormattedMessage } from 'react-intl'
+
+import { deleteContract } from '../redux'
+
+import DeleteModal from '../components/DeleteModal'
+import DeleteStatus from '../components/DeleteStatus'
+
+const ContractRemoveButton = ({ contract, deleteContract }) => {
+  if (!contract || contract.is_read_only) {
+    return ''
+  }
+
+  const [showDeleteModal, setShowDeleteModal] = useState(false)
+  const [showDeleteProgress, setShowDeleteProgress] = useState(false)
+  const [deleteOptions, setDeleteOptions] = useState()
+
+  const handleDelete = (options) => {
+    deleteContract(contract.id, deleteOptions)
+    setDeleteOptions(options)
+    setShowDeleteModal(false)
+    setShowDeleteProgress(true)
+  }
+
+  return (
+    <>
+      <button
+        className='btn btn-d btn-red btn-big'
+        onClick={() => { setShowDeleteModal(true) }}
+      >
+        <span className='details-title'>
+          <FormattedMessage
+            id='contract.remove.title'
+            defaultMessage='Delete Contract'
+          />
+        </span>
+      </button>
+
+      {
+        showDeleteModal &&
+          <DeleteModal
+            onClose={() => { setShowDeleteModal(false) }}
+            onDelete={(options) => { handleDelete(options) }}
+            objectType='contract'
+            obj={contract}
+          />
+      }
+
+      {
+        showDeleteProgress &&
+          <DeleteStatus
+            header={
+              <FormattedMessage
+                id='contract.remove.status.header'
+                defaultMessage='Deleting contract '
+              />
+            }
+            deleteOptions={deleteOptions}
+            toggle={() => { setShowDeleteProgress(false) }}
+            showModal={showDeleteProgress}
+          />
+      }
+    </>
+  )
+}
+
+const mapStateToProps = ({ pipelines }) => ({
+  contract: pipelines.currentContract
+})
+const mapDispatchToProps = { deleteContract }
+
+export default connect(mapStateToProps, mapDispatchToProps)(ContractRemoveButton)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/ContractTabs.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractTabs.jsx
new file mode 100644
index 000000000..ee7b83a48
--- /dev/null
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/ContractTabs.jsx
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+ *
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react'
+import { connect } from 'react-redux'
+import { FormattedMessage } from 'react-intl'
+
+import ContractAddButton from './ContractAddButton'
+import { selectContract } from '../redux'
+
+const ContractTabs = ({
+  isAddingNew,
+  checkUnsavedContract,
+  current,
+  list,
+  pipeline,
+  selectContract,
+  showSettings,
+  toggleSettings
+}) => {
+  return (
+    <div className='pipeline-tabs'>
+      {
+        list.map(item => (
+          <div
+            key={item.id}
+            className={`pipeline-tab ${!isAddingNew && item.id === current.id ? 'active' : ''}`}
+            onClick={() => {
+              checkUnsavedContract(() => { selectContract(pipeline.id, item.id) })
+            }}
+          >
+            <span className='contract-name'>{item.name}</span>
+
+            {
+              (item.mapping_errors || []).length > 0 &&
+                <span className={`status ${(item.mapping_errors || []).length ? 'red' : 'green'}`} />
+            }
+
+            <div
+              className={`btn-icon settings-button ${showSettings ? 'active' : ''}`}
+              onClick={() => { toggleSettings() }}
+            >
+              <i className='fas fa-wrench' />
+            </div>
+          </div>
+        ))
+      }
+
+      {
+        isAddingNew &&
+          <div key='new-contract' className='pipeline-tab active'>
+            <span className='contract-name new'>
+              <FormattedMessage
+                id='contracts.tab.new.contract'
+                defaultMessage='New contract'
+              />
+            </span>
+          </div>
+      }
+
+      {
+        !isAddingNew &&
+          <ContractAddButton
+            className='btn btn-c btn-sm new-contract'
+            pipeline={pipeline}
+          />
+      }
+    </div>
+  )
+}
+
+const mapStateToProps = ({ pipelines }) => ({
+  isAddingNew: !!pipelines.newContract,
+  current: pipelines.currentContract,
+  list: pipelines.currentPipeline.contracts,
+  pipeline: pipelines.currentPipeline
+})
+
+const mapDispatchToProps = { selectContract }
+
+export default connect(mapStateToProps, mapDispatchToProps)(ContractTabs)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteModal.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteModal.jsx
index d9b322b30..2ca5a7ef4 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteModal.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteModal.jsx
@@ -20,6 +20,7 @@
 
 import React, { useState } from 'react'
 import { FormattedMessage, defineMessages, useIntl } from 'react-intl'
+
 import { Modal } from '../../components'
 
 const MESSAGES = defineMessages({
@@ -54,7 +55,7 @@ const DeleteModal = ({ onClose, onDelete, obj, objectType }) => {
   const buttons = (
     <div>
       <button
-        data-qa='delete.modal.button.cancel'
+        data-test='delete.modal.button.cancel'
         className='btn btn-w'
         onClick={onClose}
       >
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
index 59b49ba86..617e20b38 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
@@ -19,10 +19,11 @@
  */
 
 import React, { useEffect } from 'react'
-import { Modal } from '../../components'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
 
+import { Modal } from '../../components'
+
 const DeleteStatus = ({
   deleteOptions,
   deleteStatus,
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/IdentityContract.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/IdentityContract.jsx
new file mode 100644
index 000000000..ba18567a2
--- /dev/null
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/IdentityContract.jsx
@@ -0,0 +1,180 @@
+/*
+ * Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+ *
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useState } from 'react'
+import { FormattedMessage } from 'react-intl'
+
+import { Modal } from '../../components'
+import { isEmpty } from '../../utils'
+import { deriveEntityTypes, deriveMappingRules } from '../../utils/avro-utils'
+
+export const IdentityContract = ({ contract, inputSchema, setIdentity }) => {
+  if (isEmpty(contract) || contract.is_read_only || isEmpty(inputSchema)) {
+    return ''
+  }
+
+  if (contract.is_identity) {
+    return (
+      <div data-test='contract.is-identity' className='identity-contract'>
+        <h5>
+          <FormattedMessage
+            id='identity.contract.is.label.info'
+            defaultMessage='This is an identity contract'
+          />
+        </h5>
+        <p>
+          <FormattedMessage
+            id='identity.contract.is.label.message'
+            defaultMessage='All Entity Types and Mappings were automatically generated from the input data'
+          />
+        </p>
+      </div>
+    )
+  }
+
+  const [isIdentity, setIsIdentity] = useState(contract.is_identity || false)
+  const [entityTypeName, setEntityTypeName] = useState(inputSchema.name)
+
+  const buildIdentity = (checked) => {
+    setIsIdentity(checked)
+    if (checked) {
+      const mappingRules = deriveMappingRules(inputSchema, entityTypeName)
+      const entityTypes = deriveEntityTypes(inputSchema, entityTypeName)
+
+      const identityData = {
+        mapping_rules: mappingRules,
+        entity_types: entityTypes,
+        is_identity: true
+      }
+
+      setIdentity(identityData)
+    } else {
+      setIdentity({})
+    }
+  }
+
+  return (
+    <div data-test='contract.is-not-identity' className='identity-contract'>
+      <div className='toggle-default'>
+        <input
+          data-test='identity-toggle'
+          type='checkbox'
+          id='toggle'
+          checked={isIdentity || false}
+          onChange={(event) => { buildIdentity(event.target.checked) }}
+        />
+        <label htmlFor='toggle' className='title-medium'>
+          <FormattedMessage
+            id='identity.contract.form.label'
+            defaultMessage='Create an identity contract'
+          />
+        </label>
+      </div>
+      <p>
+        <FormattedMessage
+          id='identity.contract.form.label.definition'
+          defaultMessage='An identity contract will produce entities
+            that are identical with the input. If you choose this setting,
+            Aether will generate an Entity Type and a Mapping for you.'
+        />
+      </p>
+      <p>
+        <FormattedMessage
+          id='identity.contract.form.label.explanation'
+          defaultMessage="This can be useful in situations where you
+            want to make use of Aether's functionality without transforming
+            the data. Alternatively, you can use the generate Entity Type
+            and Mapping as a starting point for a more complex contract."
+        />
+      </p>
+      {
+        isIdentity &&
+          <div className='form-group'>
+            <label className='form-label'>
+              <FormattedMessage
+                id='identity.contract.entity.type.name'
+                defaultMessage='Entity Type name'
+              />
+            </label>
+            <input
+              data-test='identity.contract.entity.type.name'
+              type='text'
+              required
+              name='name'
+              className='input-d input-large'
+              value={entityTypeName || ''}
+              onChange={(e) => { setEntityTypeName(e.target.value) }}
+            />
+          </div>
+      }
+    </div>
+  )
+}
+
+export const IdentityWarning = ({ show, onCancel, onConfirm }) => {
+  if (!show) return ''
+
+  return (
+    <Modal
+      data-test='identity-warning'
+      onEscape={onCancel}
+      onEnter={onConfirm}
+      header={
+        <FormattedMessage
+          id='identity.warning.header'
+          defaultMessage='Create identity contract'
+        />
+      }
+      buttons={
+        <div>
+          <button
+            data-test='identity.warning.button.cancel'
+            className='btn btn-w'
+            onClick={onCancel}
+          >
+            <FormattedMessage
+              id='identity.warning.button.cancel'
+              defaultMessage='Cancel'
+            />
+          </button>
+          <button
+            data-test='identity.warning.button.confirm'
+            className='btn btn-w btn-primary'
+            onClick={onConfirm}
+          >
+            <FormattedMessage
+              id='identity.warning.button.confirm'
+              defaultMessage='Yes'
+            />
+          </button>
+        </div>
+      }
+    >
+      <FormattedMessage
+        id='identity.warning.content.question'
+        defaultMessage='Are you sure that you want to create an identity contract?'
+      />
+      <FormattedMessage
+        id='identity.warning.content.message'
+        defaultMessage='This action will overwrite any entity types and mappings that you have created in this contract.'
+      />
+    </Modal>
+  )
+}
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineActions.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineActions.jsx
index 09291c7a6..040189efb 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineActions.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineActions.jsx
@@ -30,23 +30,22 @@ const PipelineOptions = ({ pipeline }) => {
 
   return (
     <OutsideClickHandler onOutsideClick={() => { setShowOptions(false) }}>
-      <button
-        type='button'
-        className='btn btn-c btn-square mr-2'
-        onClick={() => { setShowOptions(!showOptions) }}
-      >
-        <span className='details-title'>
-          <i className='fas fa-ellipsis-h' />
-        </span>
-      </button>
+      <>
+        <button
+          type='button'
+          className='btn btn-c btn-square mr-2'
+          onClick={() => { setShowOptions(!showOptions) }}
+        >
+          <span className='details-title'>
+            <i className='fas fa-ellipsis-h' />
+          </span>
+        </button>
 
-      {
-        showOptions &&
-          <ul className='options'>
-            <PipelineRemoveButton pipeline={pipeline} />
-            <PipelineRenameButton pipeline={pipeline} />
-          </ul>
-      }
+        <ul className={`options ${showOptions ? '' : 'd-none'}`}>
+          <PipelineRemoveButton pipeline={pipeline} />
+          <PipelineRenameButton pipeline={pipeline} />
+        </ul>
+      </>
     </OutsideClickHandler>
   )
 }
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineInfo.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineInfo.jsx
index ccb1e207d..4f7beacf4 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineInfo.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineInfo.jsx
@@ -21,8 +21,8 @@
 import React from 'react'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
-
 import OutsideClickHandler from 'react-outside-click-handler'
+
 import Modal from '../../components/Modal'
 import SubmissionCard from './SubmissionCard'
 
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRemoveButton.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRemoveButton.jsx
index 76406025c..0c8296ceb 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRemoveButton.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRemoveButton.jsx
@@ -20,8 +20,8 @@
 
 import React, { useState } from 'react'
 import { FormattedMessage } from 'react-intl'
-
 import { connect } from 'react-redux'
+
 import { deletePipeline } from '../redux'
 
 import DeleteStatus from './DeleteStatus'
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRenameButton.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRenameButton.jsx
index dc253ee1d..1284f2b8e 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRenameButton.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/PipelineRenameButton.jsx
@@ -20,9 +20,9 @@
 
 import React, { useState } from 'react'
 import { FormattedMessage, defineMessages, useIntl } from 'react-intl'
-import { Modal } from '../../components'
-
 import { connect } from 'react-redux'
+
+import { Modal } from '../../components'
 import { renamePipeline } from '../redux'
 
 const MESSAGES = defineMessages({
@@ -65,7 +65,7 @@ const RenameForm = ({ initialValue, placeholder, onSave, onCancel }) => {
 
       <div className='modal-actions'>
         <button
-          data-qa='rename.modal.button.cancel'
+          data-test='rename.modal.button.cancel'
           className='btn btn-w'
           onClick={onCancel}
         >
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/SubmissionCard.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/SubmissionCard.jsx
index 2172d9268..cbc573a92 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/SubmissionCard.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/SubmissionCard.jsx
@@ -21,8 +21,8 @@
 import React, { useEffect } from 'react'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
-import { Clipboard } from '../../components'
 
+import { Clipboard } from '../../components'
 import { getKernelURL } from '../../redux/settings'
 import { objectToString } from '../../utils'
 
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/redux.jsx b/aether-ui/aether/ui/assets/apps/pipeline/redux.jsx
index cd89c398c..0535bf810 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/redux.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/redux.jsx
@@ -21,12 +21,13 @@
 // Combines types, actions and reducers for a specific
 // module in one file for easy redux management
 
-import { replaceItemInList, removeItemFromList } from '../utils'
+import { generateGUID, replaceItemInList, removeItemFromList } from '../utils'
 import {
   PIPELINES_URL,
   CONTRACTS_URL,
   PIPELINE_SECTION_INPUT,
-  CONTRACT_SECTION_ENTITY_TYPES
+  CONTRACT_SECTION_ENTITY_TYPES,
+  CONTRACT_SECTION_MAPPING
 } from '../utils/constants'
 
 export const types = {
@@ -40,13 +41,12 @@ export const types = {
   PIPELINE_SELECT: 'pipeline.select',
   CONTRACT_SELECT: 'contract.select',
   SECTION_SELECT: 'section.select',
-  CONTRACT_CHANGED: 'contract.changed',
 
   PIPELINE_ADD: 'pipeline.add',
   PIPELINE_UPDATE: 'pipeline.update',
   PIPELINE_DELETE: 'pipeline.delete',
-  PIPELINE_CHANGED: 'pipeline.changed',
 
+  CONTRACT_NEW: 'contract.new',
   CONTRACT_ADD: 'contract.add',
   CONTRACT_UPDATE: 'contract.update',
   CONTRACT_DELETE: 'contract.delete',
@@ -71,7 +71,8 @@ export const INITIAL_STATE = {
 
   currentSection: null,
   currentPipeline: null,
-  currentContract: null
+  currentContract: null,
+  newContract: null
 }
 
 export const getPipelines = () => ({
@@ -91,9 +92,9 @@ export const selectPipeline = (pid) => ({
   payload: pid
 })
 
-export const selectContract = (pid, cid) => ({
+export const selectContract = (pid, cid, section = null) => ({
   type: types.CONTRACT_SELECT,
-  payload: { pipeline: pid, contract: cid }
+  payload: { pipeline: pid, contract: cid, section }
 })
 
 export const selectSection = (section) => ({
@@ -101,14 +102,9 @@ export const selectSection = (section) => ({
   payload: section
 })
 
-export const contractChanged = (contract) => ({
-  type: types.CONTRACT_CHANGED,
-  payload: contract
-})
-
-export const pipelineChanged = (pipeline) => ({
-  type: types.PIPELINE_CHANGED,
-  payload: pipeline
+export const startNewContract = (pid = null) => ({
+  type: types.CONTRACT_NEW,
+  payload: pid
 })
 
 export const getPipelineById = (pid) => {
@@ -251,6 +247,27 @@ const findContract = (pipeline, cid) => {
   return contract
 }
 
+const generateNewContractName = (contracts) => {
+  let count = 0
+  let newContractName = `Contract ${count}`
+
+  do {
+    if (!contracts.find(c => c.name === newContractName)) {
+      return newContractName
+    }
+    count++
+    newContractName = `Contract ${count}`
+  } while (true)
+}
+
+const createNewContractSkeleton = (pipeline) => {
+  return {
+    id: generateGUID(),
+    name: generateNewContractName(pipeline.contracts),
+    pipeline: pipeline.id
+  }
+}
+
 const reducer = (state = INITIAL_STATE, action) => {
   const nextState = { ...state, ...ACTIONS_INITIAL_STATE }
 
@@ -281,7 +298,8 @@ const reducer = (state = INITIAL_STATE, action) => {
         ...nextState,
         currentSection: null,
         currentPipeline: null,
-        currentContract: null
+        currentContract: null,
+        newContract: null
       }
     }
 
@@ -293,7 +311,8 @@ const reducer = (state = INITIAL_STATE, action) => {
         ...nextState,
         currentSection: PIPELINE_SECTION_INPUT,
         currentPipeline,
-        currentContract
+        currentContract,
+        newContract: null
       }
     }
 
@@ -301,15 +320,18 @@ const reducer = (state = INITIAL_STATE, action) => {
       const currentPipeline = (state.pipelinesList || [])
         .find(p => p.id === action.payload.pipeline) || state.currentPipeline
       const currentContract = findContract(currentPipeline, action.payload.contract)
-      const currentSection = !state.currentSection || state.currentSection === PIPELINE_SECTION_INPUT
-        ? CONTRACT_SECTION_ENTITY_TYPES
-        : state.currentSection
+      const currentSection = action.payload.section
+        ? action.payload.section
+        : !state.currentSection || state.currentSection === PIPELINE_SECTION_INPUT
+          ? currentContract.is_identity ? CONTRACT_SECTION_MAPPING : CONTRACT_SECTION_ENTITY_TYPES
+          : state.currentSection
 
       return {
         ...nextState,
         currentSection,
         currentPipeline,
-        currentContract
+        currentContract,
+        newContract: null
       }
     }
 
@@ -322,15 +344,6 @@ const reducer = (state = INITIAL_STATE, action) => {
 
     // CHANGES
 
-    case types.PIPELINE_CHANGED: {
-      const currentPipeline = parsePipeline(action.payload)
-      return {
-        ...nextState,
-        pipelinesList: replaceItemInList(state.pipelinesList, currentPipeline),
-        currentPipeline
-      }
-    }
-
     case types.PIPELINE_ADD: {
       const newPipeline = parsePipeline(action.payload)
 
@@ -351,7 +364,8 @@ const reducer = (state = INITIAL_STATE, action) => {
         ...nextState,
         pipelinesList: replaceItemInList(state.pipelinesList, currentPipeline),
         currentPipeline,
-        currentContract
+        currentContract,
+        newContract: null
       }
     }
 
@@ -361,10 +375,25 @@ const reducer = (state = INITIAL_STATE, action) => {
         pipelinesList: removeItemFromList(state.pipelinesList, state.currentPipeline),
         currentPipeline: null,
         currentContract: null,
+        newContract: null,
         deleteStatus: action.payload
       }
     }
 
+    case types.CONTRACT_NEW: {
+      const currentPipeline = action.payload
+        ? (state.pipelinesList || []).find(p => p.id === action.payload) || state.currentPipeline
+        : state.currentPipeline
+      const newContract = createNewContractSkeleton(currentPipeline)
+
+      return {
+        ...nextState,
+        currentSection: CONTRACT_SECTION_ENTITY_TYPES,
+        currentPipeline,
+        newContract
+      }
+    }
+
     case types.CONTRACT_ADD: {
       const currentContract = parseContract(action.payload)
       const currentPipeline = (state.pipelinesList || [])
@@ -374,55 +403,38 @@ const reducer = (state = INITIAL_STATE, action) => {
       return {
         ...nextState,
         pipelinesList: replaceItemInList(state.pipelinesList, currentPipeline),
+        currentSection: currentContract.is_identity ? CONTRACT_SECTION_MAPPING : CONTRACT_SECTION_ENTITY_TYPES,
         currentPipeline,
-        currentContract
+        currentContract,
+        newContract: null
       }
     }
 
     case types.CONTRACT_UPDATE: {
       const currentContract = parseContract(action.payload)
-      const uPipeline = { ...state.currentPipeline }
-      uPipeline.contracts = replaceItemInList(state.currentPipeline.contracts, currentContract)
+      const currentPipeline = { ...state.currentPipeline }
+      currentPipeline.contracts = replaceItemInList(state.currentPipeline.contracts, currentContract)
 
       return {
         ...state,
-        pipelinesList: replaceItemInList(state.pipelinesList, uPipeline),
-        currentPipeline: uPipeline,
-        currentContract: currentContract
+        pipelinesList: replaceItemInList(state.pipelinesList, currentPipeline),
+        currentPipeline,
+        currentContract
       }
     }
 
     case types.CONTRACT_DELETE: {
-      const uPipeline = { ...state.currentPipeline }
-      uPipeline.contracts = removeItemFromList(state.currentPipeline.contracts, state.currentContract)
-      const currentContract = uPipeline.contracts[0]
-
-      return {
-        ...state,
-        pipelinesList: replaceItemInList(state.pipelinesList, uPipeline),
-        currentPipeline: uPipeline,
-        currentContract: currentContract,
-        deleteStatus: action.payload,
-        currentSection: currentContract ? CONTRACT_SECTION_ENTITY_TYPES : PIPELINE_SECTION_INPUT
-      }
-    }
-
-    case types.CONTRACT_CHANGED: {
-      const currentContract = parseContract(action.payload)
-      let currentPipeline = state.currentPipeline
-      if (currentPipeline.id !== currentContract.pipeline) {
-        currentPipeline = (state.pipelinesList || []).find(p => p.id === currentContract.pipeline)
-        if (!currentPipeline) {
-          currentPipeline = state.currentPipeline
-          currentContract.pipeline = currentPipeline.id
-        }
-      }
+      const currentPipeline = { ...state.currentPipeline }
+      currentPipeline.contracts = removeItemFromList(state.currentPipeline.contracts, state.currentContract)
+      const currentContract = currentPipeline.contracts.length ? currentPipeline.contracts[0] : null
 
       return {
         ...state,
         pipelinesList: replaceItemInList(state.pipelinesList, currentPipeline),
+        currentSection: currentContract ? state.currentSection : PIPELINE_SECTION_INPUT,
+        currentPipeline,
         currentContract,
-        currentPipeline
+        deleteStatus: action.payload
       }
     }
 
@@ -463,8 +475,10 @@ const reducer = (state = INITIAL_STATE, action) => {
       return {
         ...nextState,
         error: action.error,
+        currentSection: null,
         currentPipeline: null,
-        currentContract: null
+        currentContract: null,
+        newContract: null
       }
     }
 
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/redux.spec.jsx b/aether-ui/aether/ui/assets/apps/pipeline/redux.spec.jsx
index 48e6fce42..8513e0ff0 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/redux.spec.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/redux.spec.jsx
@@ -175,15 +175,16 @@ describe('Pipeline actions', () => {
 
         expect(store.getState().currentPipeline.id).toEqual(1)
         expect(store.getState().currentContract.id).toEqual('1-1')
+        expect(store.getState().currentSection).toEqual('input')
       })
   })
 
   it('should dispatch a select contract action and update the redux store', () => {
     const expectedAction = {
       type: types.CONTRACT_SELECT,
-      payload: { pipeline: 2, contract: '2-1' }
+      payload: { pipeline: 2, contract: '2-1', section: 'section' }
     }
-    expect(selectContract(2, '2-1')).toEqual(expectedAction)
+    expect(selectContract(2, '2-1', 'section')).toEqual(expectedAction)
 
     nock('http://localhost')
       .post('/api/pipelines/fetch/')
@@ -194,12 +195,13 @@ describe('Pipeline actions', () => {
         expect(store.getState().currentPipeline).toBeFalsy()
         expect(store.getState().currentContract).toBeFalsy()
 
-        store.dispatch(selectPipeline(2, '2-1'))
+        store.dispatch(selectContract(2, '2-1', 'section'))
         expect(store.getState().currentPipeline).toBeTruthy()
         expect(store.getState().currentContract).toBeTruthy()
 
         expect(store.getState().currentPipeline.id).toEqual(2)
         expect(store.getState().currentContract.id).toEqual('2-1')
+        expect(store.getState().currentSection).toEqual('section')
       })
   })
 
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx b/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
index 33050ba12..8b6c4c7dc 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
@@ -143,7 +143,7 @@ const Input = ({ pipeline, highlight, updatePipeline }) => {
         // TODO: do not generate a new schema if the current one conforms the input
         const schema = generateSchema(obj)
         // Take pipeline name and remove forbidden characters
-        const name = pipeline.name.replace(/[^a-zA-Z0-9]/g, '')
+        const name = pipeline.name.replace(/[^a-zA-Z0-9]/g, '').replace(/^[-\d\s]*/g, '')
         schema.name = name.substring(0, 25)
 
         updatePipeline({ ...pipeline, schema, input: obj })
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/sections/Sections.jsx b/aether-ui/aether/ui/assets/apps/pipeline/sections/Sections.jsx
new file mode 100644
index 000000000..8a1c8d6b5
--- /dev/null
+++ b/aether-ui/aether/ui/assets/apps/pipeline/sections/Sections.jsx
@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+ *
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react'
+import { connect } from 'react-redux'
+import { FormattedMessage } from 'react-intl'
+import { useHistory } from 'react-router-dom'
+
+import { Fullscreen } from '../../components'
+
+import Input from './Input'
+import EntityTypes from './EntityTypes'
+import Mapping from './Mapping'
+import Output from './Output'
+
+import {
+  PIPELINE_SECTION_INPUT,
+  CONTRACT_SECTION_ENTITY_TYPES,
+  CONTRACT_SECTION_MAPPING
+} from '../../utils/constants'
+
+import { selectSection, startNewContract } from '../redux'
+
+const Sections = ({
+  checkUnsavedContract,
+  pipeline,
+  contract,
+  selectSection,
+  fullscreen,
+  toggleFullscreen,
+  toggleOutput,
+  startNewContract
+}) => {
+  if (!pipeline) return ''
+
+  const history = useHistory()
+  const showInput = () => {
+    checkUnsavedContract(() => { selectSection(PIPELINE_SECTION_INPUT) })
+  }
+  const showContracts = () => {
+    if (pipeline.contracts.length === 0) {
+      startNewContract(pipeline.id)
+      history.push(`/${pipeline.id}`)
+    } else {
+      selectSection(CONTRACT_SECTION_ENTITY_TYPES)
+    }
+  }
+
+  return (
+    <>
+      <div className='pipeline-nav'>
+        <div className='pipeline-nav-items'>
+          <div className='pipeline-nav-item--input' onClick={showInput}>
+            <div className='badge'>
+              <i className='fas fa-file' />
+            </div>
+            <FormattedMessage
+              id='sections.bar.input'
+              defaultMessage='Input'
+            />
+          </div>
+
+          <div className='pipeline-nav-item--contracts' onClick={showContracts}>
+            <div className='badge'>
+              <i className='fas fa-caret-right' />
+            </div>
+            <FormattedMessage
+              id='sections.bar.contracts'
+              defaultMessage='Contracts'
+            />
+          </div>
+
+          <div
+            className='pipeline-nav-item--entities'
+            onClick={() => { selectSection(CONTRACT_SECTION_ENTITY_TYPES) }}
+          >
+            <div className='badge'>
+              <i className='fas fa-caret-right' />
+            </div>
+            <FormattedMessage
+              id='sections.bar.entity.types'
+              defaultMessage='Entity Types'
+            />
+          </div>
+
+          <div
+            className='pipeline-nav-item--mapping'
+            onClick={() => { selectSection(CONTRACT_SECTION_MAPPING) }}
+          >
+            <div className='badge'>
+              <i className='fas fa-caret-right' />
+            </div>
+            <FormattedMessage
+              id='sections.bar.mapping'
+              defaultMessage='Mapping'
+            />
+          </div>
+        </div>
+
+        <div className='pipeline-nav-item--output' onClick={() => { toggleOutput() }}>
+          <div className='badge'>
+            <i className='fas fa-caret-right' />
+          </div>
+          <FormattedMessage
+            id='sections.bar.output'
+            defaultMessage='Output'
+          />
+          {
+            ((contract && contract.mapping_errors) || []).length > 0 &&
+              <span className={`status ${(contract.mapping_errors || []).length ? 'red' : 'green'}`} />
+          }
+        </div>
+      </div>
+
+      <div className='pipeline-sections'>
+        <div className='pipeline-section--input'><Input /></div>
+        {
+          contract &&
+            <>
+              <div className='pipeline-section--entities'>
+                <EntityTypes />
+                <Fullscreen value={fullscreen} toggle={toggleFullscreen} />
+              </div>
+              <div className='pipeline-section--mapping'>
+                <Mapping />
+                <Fullscreen value={fullscreen} toggle={toggleFullscreen} />
+              </div>
+            </>
+        }
+      </div>
+      {contract && <div className='pipeline-output'><Output /></div>}
+    </>
+  )
+}
+
+const mapStateToProps = ({ pipelines }) => ({
+  pipeline: pipelines.currentPipeline,
+  contract: pipelines.currentContract
+})
+const mapDispatchToProps = { selectSection, startNewContract }
+
+export default connect(mapStateToProps, mapDispatchToProps)(Sections)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.jsx b/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.jsx
index f3d317f00..723b1b3a7 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.jsx
@@ -18,379 +18,156 @@
  * under the License.
  */
 
-import React, { Component } from 'react'
+import React, { useState, useEffect } from 'react'
 import { connect } from 'react-redux'
 import { FormattedMessage } from 'react-intl'
 
-import { updateContract, addContract, contractChanged } from '../redux'
-import { isEmpty, generateGUID } from '../../utils'
-import { deriveEntityTypes, deriveMappingRules } from '../../utils/avro-utils'
+import { addContract, updateContract } from '../redux'
 
-import { Modal } from '../../components'
 import ContractPublishButton from '../components/ContractPublishButton'
+import ContractRemoveButton from '../components/ContractRemoveButton'
 import SubmissionCard from '../components/SubmissionCard'
-
-import { CONTRACT_SECTION_MAPPING } from '../../utils/constants'
-
-const generateNewContractName = (pipeline) => {
-  let count = 0
-  let newContractName = `Contract ${count}`
-
-  do {
-    if (!pipeline.contracts.find(c => c.name === newContractName)) {
-      return newContractName
+import { IdentityContract, IdentityWarning } from '../components/IdentityContract'
+
+const Settings = ({
+  addContract,
+  contract,
+  onClose,
+  onSave,
+  pipeline,
+  updateContract
+}) => {
+  if (!contract) return ''
+
+  const [prevContract, setPrevContract] = useState(contract)
+  const [contractName, setContractName] = useState(contract.name)
+  const [identity, setIdentity] = useState({})
+  const [showIdentityWarning, setShowIdentityWarning] = useState(false)
+
+  useEffect(() => {
+    if (prevContract !== contract) {
+      setPrevContract(contract)
+      setContractName(contract.name)
     }
-    count++
-    newContractName = `Contract ${count}`
-  } while (true)
-}
-
-export class IdentityMapping extends Component {
-  constructor (props) {
-    super(props)
+  })
 
-    this.state = {
-      entityTypeName: props.inputSchema.name || ''
+  const handlePreSave = () => {
+    if (contract.created && identity.is_identity) {
+      setShowIdentityWarning(true)
+    } else {
+      handleSave()
     }
   }
 
-  render () {
-    const { contract = {} } = this.props
-    return (
-      <>
-        {
-          !contract.is_identity &&
-            <div className='identity-mapping'>
-              <div className='toggle-default'>
-                <input
-                  type='checkbox'
-                  id='toggle'
-                  checked={this.props.isIdentity}
-                  onChange={(e) => { this.props.onChange(e) }}
-                />
-                <label
-                  htmlFor='toggle'
-                  className='title-medium'
-                >
-                  <FormattedMessage
-                    id='settings.identity.help-1'
-                    defaultMessage='Create an identity contract'
-                  />
-                </label>
-              </div>
-              <p>
-                <FormattedMessage
-                  id='settings.identity.help-2'
-                  defaultMessage='An identity contract will produce entities
-                    that are identical with the input. If you choose this setting,
-                    Aether will generate an Entity Type and Mapping for you.'
-                />
-              </p>
-              <p>
-                <FormattedMessage
-                  id='settings.identity.help-3'
-                  defaultMessage="This can be useful in situations where you
-                    want to make use of Aether's functionality without transforming
-                    the data. Alternatively, you can use the generate Entity Type
-                    and Mapping as a starting point for a more complex contract."
-                />
-              </p>
-              {
-                this.props.isIdentity &&
-                  <div className='form-group'>
-                    <label className='form-label'>
-                      <FormattedMessage
-                        id='settings.contract.identity.name'
-                        defaultMessage='Entity Type name'
-                      />
-                    </label>
-                    <input
-                      type='text'
-                      required
-                      name='name'
-                      className='input-d input-large'
-                      value={this.state.entityTypeName}
-                      onChange={(e) => { this.setState({ entityTypeName: e.target.value }) }}
-                    />
-                  </div>
-              }
-            </div>
-        }
-
-        {
-          contract.is_identity &&
-            <div className='identity-mapping'>
-              <h5>
-                <FormattedMessage
-                  id='settings.identity.checked.help-1'
-                  defaultMessage='This is an identity contract'
-                />
-              </h5>
-              <p>
-                <FormattedMessage
-                  id='settings.identity.checked.help-2'
-                  defaultMessage='All Entity Types and Mappings were automatically generated from the input data'
-                />
-              </p>
-            </div>
-        }
-
-        {this.props.showModal && this.renderModal()}
-      </>
-    )
+  const handleSave = () => {
+    setShowIdentityWarning(false)
+    const updatedContract = { ...contract, ...identity, name: contractName }
+    const save = updatedContract.created ? updateContract : addContract
+    save(updatedContract)
+    onSave()
   }
 
-  renderModal () {
-    const header = (
-      <FormattedMessage
-        id='settings.identity.header'
-        defaultMessage='Create identity contract'
-      />
-    )
-
-    const handleGenerateIdentityMapping = () => {
-      const schema = this.props.inputSchema
-      const mappingRules = deriveMappingRules(schema, this.state.entityTypeName)
-      const entityTypes = deriveEntityTypes(schema, this.state.entityTypeName)
-
-      const updatedContract = {
-        ...this.props.contract,
-        mapping_rules: mappingRules,
-        entity_types: entityTypes,
-        name: this.props.contractName,
-        is_identity: true
-      }
-
-      this.props.onSave(updatedContract)
-      this.props.onModalToggle(false)
-    }
-
-    const close = () => { this.props.onModalToggle(false) }
-    const buttons = (
-      <div>
-        <button
-          id='settings.identity.modal.cancel'
-          className='btn btn-w'
-          onClick={close}
-        >
-          <FormattedMessage
-            id='settings.identity.button.cancel'
-            defaultMessage='Cancel'
-          />
-        </button>
-        <button
-          data-qa='contract.identity.button.confirm'
-          className='btn btn-w btn-primary'
-          id='settings.identity.modal.yes'
-          onClick={handleGenerateIdentityMapping}
-        >
-          <FormattedMessage
-            id='settings.identity.button.confirm'
-            defaultMessage='Yes'
+  return (
+    <div data-test='contract-settings' className='pipeline-settings'>
+      <div className='contract-form'>
+        <div className='form-group'>
+          <label className='form-label'>
+            <FormattedMessage
+              id='settings.contract.name'
+              defaultMessage='Contract name'
+            />
+          </label>
+          <input
+            data-test='settings.contract.name'
+            type='text'
+            required
+            name='name'
+            className='input-d input-large'
+            value={contractName || ''}
+            onChange={(e) => { setContractName(e.target.value) }}
+            disabled={contract.is_read_only}
           />
-        </button>
-      </div>
-    )
+        </div>
 
-    return (
-      <Modal
-        header={header}
-        buttons={buttons}
-        onEscape={close}
-        onEnter={handleGenerateIdentityMapping}
-      >
-        <FormattedMessage
-          id='settings.identity.content.question'
-          defaultMessage='Are you sure that you want to create an identity contract?'
+        <IdentityContract
+          key={contract.id}
+          contract={contract}
+          inputSchema={pipeline.schema}
+          setIdentity={setIdentity}
         />
-        <FormattedMessage
-          id='settings.identity.content.warning'
-          defaultMessage='This action will overwrite any entity types and mappings that you have created in this contract.'
-        />
-      </Modal>
-    )
-  }
-}
-
-class Settings extends Component {
-  constructor (props) {
-    super(props)
-
-    this.state = {
-      contractName: props.contract ? props.contract.name : '',
-      isIdentity: false,
-      showIdentityModal: false
-    }
-
-    if (props.isNew) {
-      this.createNewContract()
-    }
-
-    if (props.isNew) {
-      this.createNewContract()
-    }
-
-    this.handleSave = this.handleSave.bind(this)
-    this.onSave = this.onSave.bind(this)
-  }
-
-  componentDidUpdate (prevProps) {
-    if (this.props.isNew && !prevProps.isNew) {
-      this.createNewContract()
-    }
-    if (this.props.contract && prevProps.contract !== this.props.contract) {
-      this.setState({ contractName: this.props.contract.name })
-    }
-  }
-
-  createNewContract () {
-    const newContract = {
-      name: generateNewContractName(this.props.pipeline),
-      id: generateGUID(),
-      pipeline: this.props.pipeline.id,
-      mapping_errors: []
-    }
-    this.props.contractChanged(newContract)
-    this.props.onNew(newContract)
-  }
-
-  onSave (contract) {
-    if (this.state.isIdentity) {
-      this.setState({
-        showIdentityModal: true
-      })
-    } else {
-      this.handleSave({ ...contract, name: this.state.contractName })
-    }
-  }
 
-  handleSave (contract) {
-    if (this.props.isNew) {
-      this.props.addContract(contract)
-      this.props.onSave(this.state.isIdentity ? CONTRACT_SECTION_MAPPING : null)
-    } else {
-      this.props.updateContract({
-        ...contract,
-        name: this.state.contractName
-      })
-      this.props.onClose()
-    }
-  }
-
-  render () {
-    const { contract = {} } = this.props
-    const showIdentityOption = (!contract.is_read_only && !isEmpty(this.props.inputSchema))
+        <IdentityWarning
+          show={showIdentityWarning}
+          onCancel={() => { setShowIdentityWarning(false) }}
+          onConfirm={() => { handleSave() }}
+        />
 
-    return (
-      <div className='pipeline-settings'>
-        <div className='contract-form'>
-          <div className='form-group'>
-            <label className='form-label'>
-              <FormattedMessage
-                id='settings.contract.name'
-                defaultMessage='Contract name'
-              />
-            </label>
-            <input
-              type='text'
-              required
-              name='name'
-              className='input-d input-large'
-              value={this.state.contractName}
-              onChange={(e) => { this.setState({ contractName: e.target.value }) }}
-              disabled={contract.is_read_only}
+        <div className='settings-section'>
+          <div>
+            <ContractPublishButton
+              contract={contract}
+              className='btn btn-d btn-publish'
             />
           </div>
 
           {
-            showIdentityOption &&
-              <IdentityMapping
-                {...this.props}
-                onChange={(e) => this.setState({ isIdentity: e.target.checked })}
-                isIdentity={this.state.isIdentity}
-                showModal={this.state.showIdentityModal}
-                onModalToggle={(e) => { this.setState({ showIdentityModal: e }) }}
-                contractName={this.state.contractName}
-                onSave={this.handleSave}
-              />
+            contract.published_on &&
+              <div className='mt-4'>
+                <SubmissionCard
+                  mappingset={pipeline.mappingset}
+                  inputData={pipeline.input}
+                />
+              </div>
           }
-
-          <div className='settings-section'>
-            <div>
-              <ContractPublishButton
-                contract={contract}
-                className='btn btn-d btn-publish'
+        </div>
+        <div className='settings-actions'>
+          <button
+            data-test='settings.cancel.button'
+            type='button'
+            className='btn btn-d btn-big'
+            onClick={() => { onClose() }}
+          >
+            <span className='details-title'>
+              <FormattedMessage
+                id='settings.button.cancel'
+                defaultMessage='Cancel'
               />
-            </div>
-
-            {
-              contract.published_on &&
-                <div className='mt-4'>
-                  <SubmissionCard
-                    mappingset={this.props.mappingset}
-                    inputData={this.props.inputData}
+            </span>
+          </button>
+          {
+            !contract.is_read_only &&
+              <button
+                data-test='settings.save.button'
+                type='button'
+                className='btn btn-d btn-primary btn-big ml-4'
+                onClick={() => { handlePreSave() }}
+              >
+                <span className='details-title'>
+                  <FormattedMessage
+                    id='settings.button.save'
+                    defaultMessage='Save'
                   />
-                </div>
-            }
-          </div>
-          <div className='settings-actions'>
-            <button
-              onClick={() => { this.props.onClose() }}
-              type='button'
-              className='btn btn-d btn-big'
-              id='pipeline.settings.cancel.button'
-            >
-              <span className='details-title'>
-                <FormattedMessage
-                  id='settings.button.cancel'
-                  defaultMessage='Cancel'
-                />
-              </span>
-            </button>
-            {
-              !contract.is_read_only &&
-                <button
-                  id='settings-contract-save'
-                  className='btn btn-d btn-primary btn-big ml-4'
-                  onClick={() => { this.onSave(contract) }}
-                >
-                  <span className='details-title'>
-                    <FormattedMessage
-                      id='settings.contract.save'
-                      defaultMessage='Save'
-                    />
-                  </span>
-                </button>
-            }
-            {
-              !contract.is_read_only && contract.created &&
-                <button
-                  className='btn btn-d btn-red btn-big'
-                  onClick={() => { this.props.onDelete() }}
-                >
-                  <span className='details-title'>
-                    <FormattedMessage
-                      id='settings.contract.delete'
-                      defaultMessage='Delete Contract'
-                    />
-                  </span>
-                </button>
-            }
-          </div>
+                </span>
+              </button>
+          }
+
+          <ContractRemoveButton />
         </div>
       </div>
-    )
-  }
+    </div>
+  )
 }
 
-const mapStateToProps = ({ pipelines }) => ({
-  mappingset: pipelines.currentPipeline && pipelines.currentPipeline.mappingset,
-  inputData: pipelines.currentPipeline && pipelines.currentPipeline.input,
-  inputSchema: pipelines.currentPipeline && pipelines.currentPipeline.schema,
-
-  contract: pipelines.currentContract,
-  pipeline: pipelines.currentPipeline
+const mapStateToProps = ({
+  pipelines: {
+    currentPipeline,
+    currentContract,
+    newContract
+  }
+}) => ({
+  contract: newContract || currentContract,
+  pipeline: currentPipeline
 })
-const mapDispatchToProps = { updateContract, addContract, contractChanged }
+const mapDispatchToProps = { addContract, updateContract }
 
 export default connect(mapStateToProps, mapDispatchToProps)(Settings)
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.spec.jsx b/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.spec.jsx
index 557d3f709..f3a61c539 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.spec.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/sections/Settings.spec.jsx
@@ -18,256 +18,290 @@
  * under the License.
  */
 
-/* global describe, it, expect, beforeEach, jest */
+/* global describe, it, expect, afterEach, beforeEach, jest */
 
 import React from 'react'
 import { createStore, applyMiddleware } from 'redux'
 import { Provider } from 'react-redux'
-import { mountWithIntl } from '../../../tests/enzyme-helpers'
 import nock from 'nock'
 
+import { mountWithIntl } from '../../../tests/enzyme-helpers'
+import { mockInputSchema } from '../../../tests/mock'
+
 import Settings from './Settings'
-import { mockPipeline } from '../../../tests/mock'
 import middleware from '../../redux/middleware'
 import reducer from '../../redux/reducers'
-import { contractChanged } from '../redux'
-
-describe('Pipeline Settings Component', () => {
-  let store
-  const initialState = {
-    pipelines: {
-      currentPipeline: mockPipeline
+import { selectContract } from '../redux'
+
+const INITIAL_STATE = {
+  pipelines: {
+    currentPipeline: {
+      id: 'pid',
+      name: 'pipeline',
+      schema: mockInputSchema, // no identity without schema
+      contracts: [
+        {
+          id: 'non-identity',
+          name: 'Non identity',
+          is_identity: false,
+          created: '2020-01-01'
+        },
+        {
+          id: 'identity',
+          name: 'Identity',
+          is_identity: true,
+          created: '2020-01-01'
+        }
+      ]
+    },
+    newContract: {
+      id: 'new',
+      name: 'New'
     }
   }
+}
+
+describe('Pipeline Settings Component', () => {
   const onClose = jest.fn()
   const onSave = jest.fn()
-  const onNew = jest.fn()
 
-  beforeEach(() => {
+  const mountComponent = (isNew, isIdentity = false) => {
     // create a new store instance for each test
-    store = createStore(
+    const store = createStore(
       reducer,
-      initialState,
+      INITIAL_STATE,
       applyMiddleware(...middleware)
     )
-  })
+    if (!isNew) {
+      store.dispatch(selectContract('pid', isIdentity ? 'identity' : 'non-identity'))
+      expect(store.getState().newContract).toBeFalsy()
+    }
 
-  it('should render the settings component', () => {
-    const component = mountWithIntl(
+    return mountWithIntl(
       <Provider store={store}>
         <Settings
           onClose={onClose}
           onSave={onSave}
-          onNew={onNew}
         />
       </Provider>
     )
-    expect(component.find('Settings').exists()).toBeTruthy()
-    expect(store.getState().pipelines.currentContract)
-      .toEqual(undefined)
-  })
+  }
 
-  it('should render the settings of selected contract', () => {
-    const selectedContract = mockPipeline.contracts[0]
-    store.dispatch(contractChanged(selectedContract))
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    expect(component.find('[className="input-d input-large"]')
-      .html()).toContain(`value="${selectedContract.name}"`)
-  })
+  describe('on new contracts', () => {
+    let newComponent
+    beforeEach(() => {
+      newComponent = mountComponent(true)
+    })
 
-  it('should render the settings with a new contract', () => {
-    mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          isNew
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    expect(store.getState().pipelines.currentContract.name)
-      .toEqual('Contract 0')
-  })
+    afterEach(() => {
+      nock.isDone()
+      nock.cleanAll()
+    })
 
-  it('should toggle identity mapping', () => {
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          isNew
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    expect(component.find('Settings').exists()).toBeTruthy()
-    const settings = component.find('Settings')
+    it('should render the settings component', () => {
+      expect(newComponent.find('[data-test="contract-settings"]').exists())
+        .toBeTruthy()
+    })
 
-    expect(settings.state().isIdentity).toBeFalsy()
-    expect(component.find('[id="settings.contract.identity.name"]')
-      .exists()).toBeFalsy()
+    it('should close the settings component', () => {
+      const cancelButton = newComponent.find('button[data-test="settings.cancel.button"]')
+      cancelButton.simulate('click')
+      expect(onClose).toBeCalled()
+    })
 
-    const identityMappingToggle = component.find('input[id="toggle"]')
-    identityMappingToggle.simulate('change', { target: { checked: true } })
-    expect(settings.state().isIdentity).toBeTruthy()
+    it('should render the identity toggle', () => {
+      expect(newComponent.find('[data-test="settings.contract.name"]').html())
+        .toContain('value="New"')
+      expect(newComponent.find('[data-test="contract.is-identity"]').exists())
+        .toBeFalsy()
+      expect(newComponent.find('[data-test="contract.is-not-identity"]').exists())
+        .toBeTruthy()
+    })
+
+    it('should toggle identity contract', () => {
+      expect(newComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeFalsy()
+
+      const identityToggle = newComponent.find('input[data-test="identity-toggle"]')
+      identityToggle.simulate('change', { target: { checked: true } })
+      expect(newComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeTruthy()
 
-    expect(component.find('[id="settings.contract.identity.name"]')
-      .exists()).toBeTruthy()
+      identityToggle.simulate('change', { target: { checked: false } })
+      expect(newComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeFalsy()
+    })
 
-    identityMappingToggle.simulate('change', { target: { checked: false } })
-    expect(settings.state().isIdentity).toBeFalsy()
-    expect(component.find('[id="settings.contract.identity.name"]')
-      .exists()).toBeFalsy()
+    it('should save and close settings without warning', () => {
+      let newContract
+      nock('http://localhost')
+        .post('/api/contracts/', body => {
+          newContract = body
+          return body
+        })
+        .reply(201, {})
+
+      const inputContractName = newComponent.find('input[data-test="settings.contract.name"]')
+      inputContractName.simulate('change', { target: { value: 'new-contract-updated' } })
+
+      const saveButton = newComponent.find('button[data-test="settings.save.button"]')
+      saveButton.simulate('click')
+      expect(onSave).toBeCalled()
+      expect(newContract).toEqual({ id: 'new', name: 'new-contract-updated' })
+    })
   })
 
-  it('should render identity mapping warning', () => {
-    nock('http://localhost')
-      .post('/api/contracts/')
-      .reply(200, (_, reqBody) => {
-        return reqBody
-      })
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          isNew
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    expect(component.find('Settings').exists()).toBeTruthy()
-    const settings = component.find('Settings')
+  describe('on current non-identity contracts', () => {
+    let nonIdComponent
+    beforeEach(() => {
+      nonIdComponent = mountComponent(false, false)
+    })
 
-    const identityMappingToggle = component.find('input[id="toggle"]')
-    identityMappingToggle.simulate('change', { target: { checked: true } })
-    expect(settings.state().isIdentity).toBeTruthy()
+    afterEach(() => {
+      nock.isDone()
+      nock.cleanAll()
+    })
 
-    expect(component.find('Modal').exists()).toBeFalsy()
-    const saveButton = component.find('button[id="settings-contract-save"]')
-    saveButton.simulate('click')
+    it('should render the settings component', () => {
+      expect(nonIdComponent.find('[data-test="contract-settings"]').exists())
+        .toBeTruthy()
+    })
 
-    expect(component.find('Modal').exists()).toBeTruthy()
+    it('should close the settings component', () => {
+      const cancelButton = nonIdComponent.find('button[data-test="settings.cancel.button"]')
+      cancelButton.simulate('click')
+      expect(onClose).toBeCalled()
+    })
 
-    const modalCancelButton = component.find('button[id="settings.identity.modal.cancel"]')
-    modalCancelButton.simulate('click')
-    expect(component.find('Modal').exists()).toBeFalsy()
+    it('should render the identity toggle', () => {
+      expect(nonIdComponent.find('[data-test="settings.contract.name"]').html())
+        .toContain('value="Non identity"')
+      expect(nonIdComponent.find('[data-test="contract.is-identity"]').exists())
+        .toBeFalsy()
+      expect(nonIdComponent.find('[data-test="contract.is-not-identity"]').exists())
+        .toBeTruthy()
+    })
 
-    saveButton.simulate('click')
-    expect(component.find('Modal').exists()).toBeTruthy()
+    it('should toggle identity contract', () => {
+      expect(nonIdComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeFalsy()
 
-    const modalYesButton = component.find('button[id="settings.identity.modal.yes"]')
-    modalYesButton.simulate('click')
-    expect(component.find('Modal').exists()).toBeFalsy()
-  })
+      const identityToggle = nonIdComponent.find('input[data-test="identity-toggle"]')
+      identityToggle.simulate('change', { target: { checked: true } })
+      expect(nonIdComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeTruthy()
 
-  it('should save and close settings without warning', () => {
-    const selectedContract = mockPipeline.contracts[0]
-    let expectedContract
-    nock('http://localhost')
-      .put(`/api/contracts/${selectedContract.id}/`)
-      .reply(200, (_, reqBody) => {
-        expectedContract = reqBody
-        return reqBody
-      })
-
-    store.dispatch(contractChanged(selectedContract))
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    expect(component.find('Modal').exists()).toBeFalsy()
+      identityToggle.simulate('change', { target: { checked: false } })
+      expect(nonIdComponent.find('[data-test="identity.contract.entity.type.name"]').exists())
+        .toBeFalsy()
+    })
 
-    const inputContractName = component.find('input[className="input-d input-large"]')
-    inputContractName.simulate('change', { target: { value: 'contract-updated' } })
+    it('should save and close settings without warning (while non identity)', () => {
+      let updatedContract
+      nock('http://localhost')
+        .put('/api/contracts/non-identity/', body => {
+          updatedContract = body
+          return body
+        })
+        .reply(200, {})
+
+      const inputContractName = nonIdComponent.find('input[data-test="settings.contract.name"]')
+      inputContractName.simulate('change', { target: { value: 'non-contract-updated' } })
+
+      const saveButton = nonIdComponent.find('button[data-test="settings.save.button"]')
+      saveButton.simulate('click')
+      expect(onSave).toBeCalled()
+      expect(updatedContract.name).toEqual('non-contract-updated')
+      expect(updatedContract.is_identity).toBeFalsy()
+    })
 
-    const settings = component.find('Settings').instance()
-    jest.spyOn(settings, 'handleSave')
-    const saveButton = component.find('button[id="settings-contract-save"]')
-    saveButton.simulate('click')
+    it('should render identity contract warning', () => {
+      let updatedContract
+      nock('http://localhost')
+        .put('/api/contracts/non-identity/', body => {
+          updatedContract = body
+          return body
+        })
+        .reply(200, {})
 
-    expect(component.find('Modal').exists()).toBeFalsy()
-    expect(settings.handleSave).toHaveBeenCalledWith(expectedContract)
-  })
+      const saveButton = nonIdComponent.find('button[data-test="settings.save.button"]')
+      const identityToggle = nonIdComponent.find('input[data-test="identity-toggle"]')
 
-  it('should save and close settings without warning on a new contract', () => {
-    let expectedContract
-    nock('http://localhost')
-      .post('/api/contracts/')
-      .reply(200, (_, reqBody) => {
-        expectedContract = reqBody
-        return reqBody
-      })
+      identityToggle.simulate('change', { target: { checked: true } })
+      expect(nonIdComponent.find('Modal').exists()).toBeFalsy()
 
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          onSave={onSave}
-          onNew={onNew}
-          isNew
-        />
-      </Provider>
-    )
-    expect(component.find('Modal').exists()).toBeFalsy()
+      saveButton.simulate('click')
+      expect(nonIdComponent.find('Modal').exists()).toBeTruthy()
 
-    const inputContractName = component.find('input[className="input-d input-large"]')
-    inputContractName.simulate('change', { target: { value: 'new-contract-updated' } })
+      const modalCancelButton = nonIdComponent
+        .find('Modal')
+        .find('button[data-test="identity.warning.button.cancel"]')
+      expect(modalCancelButton.exists()).toBeTruthy()
 
-    const settings = component.find('Settings').instance()
-    jest.spyOn(settings, 'handleSave')
-    const saveButton = component.find('button[id="settings-contract-save"]')
-    saveButton.simulate('click')
+      modalCancelButton.simulate('click')
+      expect(nonIdComponent.find('Modal').exists()).toBeFalsy()
 
-    expect(component.find('Modal').exists()).toBeFalsy()
-    expect(settings.handleSave).toHaveBeenCalledWith(expectedContract)
-  })
+      saveButton.simulate('click')
+      expect(nonIdComponent.find('Modal').exists()).toBeTruthy()
 
-  it('should create a new contract while an existing contract is selected', () => {
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    const settingInstance = component.find('Settings').instance()
-    jest.spyOn(settingInstance, 'createNewContract')
-    component.setProps({
-      children: <Settings onClose={onClose} onSave={onSave} onNew={onNew} isNew />
+      const modalYesButton = nonIdComponent
+        .find('Modal')
+        .find('button[data-test="identity.warning.button.confirm"]')
+      expect(modalYesButton.exists()).toBeTruthy()
+
+      modalYesButton.simulate('click')
+      expect(nonIdComponent.find('Modal').exists()).toBeFalsy()
+      expect(updatedContract.is_identity).toBeTruthy()
     })
-    expect(settingInstance.createNewContract).toBeCalled()
   })
 
-  it('should close the settings component', () => {
-    const component = mountWithIntl(
-      <Provider store={store}>
-        <Settings
-          onClose={onClose}
-          onSave={onSave}
-          onNew={onNew}
-        />
-      </Provider>
-    )
-    const cancelButton = component.find('button[id="pipeline.settings.cancel.button"]')
-    cancelButton.simulate('click')
+  describe('on current identity contracts', () => {
+    let idComponent
+    beforeEach(() => {
+      idComponent = mountComponent(false, true)
+    })
+
+    afterEach(() => {
+      nock.isDone()
+      nock.cleanAll()
+    })
+
+    it('should render the settings component', () => {
+      expect(idComponent.find('[data-test="contract-settings"]').exists())
+        .toBeTruthy()
+    })
+
+    it('should close the settings component', () => {
+      const cancelButton = idComponent.find('button[data-test="settings.cancel.button"]')
+      cancelButton.simulate('click')
+      expect(onClose).toBeCalled()
+    })
 
-    expect(onClose).toBeCalled()
+    it('should not render the identity toggle', () => {
+      expect(idComponent.find('[data-test="settings.contract.name"]').html())
+        .toContain('value="Identity"')
+      expect(idComponent.find('[data-test="contract.is-identity"]').exists())
+        .toBeTruthy()
+      expect(idComponent.find('[data-test="contract.is-not-identity"]').exists())
+        .toBeFalsy()
+    })
+
+    it('should save and close settings without warning', () => {
+      let updatedContract
+      nock('http://localhost')
+        .put('/api/contracts/identity/', body => {
+          updatedContract = body
+          return body
+        })
+        .reply(200, {})
+
+      const inputContractName = idComponent.find('input[data-test="settings.contract.name"]')
+      inputContractName.simulate('change', { target: { value: 'contract-updated' } })
+
+      const saveButton = idComponent.find('button[data-test="settings.save.button"]')
+      saveButton.simulate('click')
+      expect(onSave).toBeCalled()
+      expect(updatedContract.name).toEqual('contract-updated')
+    })
   })
 })
diff --git a/aether-ui/aether/ui/assets/apps/utils/constants.jsx b/aether-ui/aether/ui/assets/apps/utils/constants.jsx
index a3de22768..ea1a682b4 100644
--- a/aether-ui/aether/ui/assets/apps/utils/constants.jsx
+++ b/aether-ui/aether/ui/assets/apps/utils/constants.jsx
@@ -38,6 +38,5 @@ export const DATE_FORMAT = 'MMMM DD, YYYY HH:mm'
 
 // pipeline/contract view sections
 export const PIPELINE_SECTION_INPUT = 'input'
-export const CONTRACT_SECTION_ENTITY_TYPES = 'entityTypes'
+export const CONTRACT_SECTION_ENTITY_TYPES = 'entities'
 export const CONTRACT_SECTION_MAPPING = 'mapping'
-export const CONTRACT_SECTION_SETTINGS = 'settings'
diff --git a/aether-ui/aether/ui/assets/apps/utils/index.jsx b/aether-ui/aether/ui/assets/apps/utils/index.jsx
index 0f01375a0..b03b95bae 100644
--- a/aether-ui/aether/ui/assets/apps/utils/index.jsx
+++ b/aether-ui/aether/ui/assets/apps/utils/index.jsx
@@ -18,6 +18,8 @@
  * under the License.
  */
 
+import { v4 as uuidv4 } from 'uuid'
+
 /**
  * Clones object.
  *
@@ -28,14 +30,7 @@ export const clone = (x) => JSON.parse(JSON.stringify(x))
 /**
  * Generates random UUID
  */
-export const generateGUID = () => {
-  const s4 = () => {
-    return Math.floor((1 + Math.random()) * 0x10000)
-      .toString(16)
-      .substring(1)
-  }
-  return `${s4()}${s4()}-${s4()}-${s4()}-${s4()}-${s4()}${s4()}${s4()}`
-}
+export const generateGUID = () => uuidv4()
 
 /**
  * Checks if the two objects are equal, comparing even nested properties.
diff --git a/aether-ui/aether/ui/assets/conf/sass-lint.yml b/aether-ui/aether/ui/assets/conf/sass-lint.yml
index fe62b6f45..ad36412b7 100644
--- a/aether-ui/aether/ui/assets/conf/sass-lint.yml
+++ b/aether-ui/aether/ui/assets/conf/sass-lint.yml
@@ -53,7 +53,7 @@ rules:
   force-element-nesting:      0
   force-pseudo-nesting:       0
 
-  class-name-format:          0
-  variable-name-format:       0
+  class-name-format:          2
+  variable-name-format:       2
 
   # see more: https://github.com/sasstools/sass-lint/tree/develop/docs/rules
diff --git a/aether-ui/aether/ui/assets/css/_pipeline-settings.scss b/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
index 78cfc5052..a5fc0a68b 100644
--- a/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
+++ b/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
@@ -18,20 +18,6 @@
  * under the License.
  */
 
-@keyframes show {
-  0% {
-    opacity: 0;
-  }
-
-  50% {
-    opacity: 0;
-  }
-
-  100% {
-    opacity: 1;
-  }
-}
-
 .pipeline-settings {
   background: lighten($dark-blue, 5);
   position: absolute;
@@ -73,7 +59,7 @@ div.code {
   margin: 3rem 2.3rem 1rem;
 }
 
-.identity-mapping {
+.identity-contract {
   margin-top: 2rem;
   padding: 1.6rem 2rem 1.2rem 4rem;
   border-radius: $border-radius * 4;
diff --git a/aether-ui/aether/ui/assets/css/_pipeline.scss b/aether-ui/aether/ui/assets/css/_pipeline.scss
index 1f69ebbed..e6413a92d 100644
--- a/aether-ui/aether/ui/assets/css/_pipeline.scss
+++ b/aether-ui/aether/ui/assets/css/_pipeline.scss
@@ -18,19 +18,11 @@
  * under the License.
  */
 
-@keyframes slide-up {
-  0%   { top: 100vh; }
-  100% { top: $navbar-height-xs; }
-}
-
-$transition-speed: .6s;
-
-$pipeline-nav-height: 4rem;
-
-$input-nav-width: 6rem;
-$entityTypes-nav-width: 11rem;
-$mapping-nav-width: 9rem;
-$output-nav-width: 10rem;
+$pipeline-nav-height:   4rem;
+$input-nav-width:       6rem;
+$entities-nav-width:    11rem;
+$mapping-nav-width:     9rem;
+$output-nav-width:      10rem;
 
 .pipeline {
   background: $white;
@@ -76,7 +68,6 @@ $output-nav-width: 10rem;
   }
 
   .settings-button {
-
     padding: 0;
     margin-left: 2rem;
     display: none;
@@ -178,7 +169,7 @@ $output-nav-width: 10rem;
 .fullscreen.pipeline {
   top: 0;
 
-  div[class^='pipeline-section__'],
+  div[class^='pipeline-section--'],
   .pipeline-output {
     height: 100vh;
   }
@@ -208,7 +199,7 @@ $output-nav-width: 10rem;
   transition: width $transition-speed;
 }
 
-div[class^='pipeline-nav-item__'] {
+div[class^='pipeline-nav-item--'] {
   @include label;
   color: $action-color-b;
   font-weight: 600;
@@ -235,7 +226,6 @@ div[class^='pipeline-nav-item__'] {
   }
 
   &:hover {
-
     .badge {
       @include bg-gradient;
       opacity: 1;
@@ -243,25 +233,25 @@ div[class^='pipeline-nav-item__'] {
   }
 }
 
-div.pipeline-nav-item__input {
+div.pipeline-nav-item--input {
   width: $input-nav-width;
 }
 
-div.pipeline-nav-item__entityTypes {
-  width: $entityTypes-nav-width;
+div.pipeline-nav-item--entities {
+  width: $entities-nav-width;
 }
 
-div.pipeline-nav-item__mapping {
+div.pipeline-nav-item--mapping {
   width: $mapping-nav-width;
 }
 
-div.pipeline-nav-item__contracts {
+div.pipeline-nav-item--contracts {
   position: absolute;
   right: 1rem;
   display: none;
 }
 
-div.pipeline-nav-item__output {
+div.pipeline-nav-item--output {
   width: $output-nav-width;
 }
 
@@ -279,7 +269,7 @@ div.pipeline-nav-item__output {
   overflow-y: auto;
 }
 
-div[class^='pipeline-section__'],
+div[class^='pipeline-section--'],
 .pipeline-output {
   width: 0;
   height: calc(100vh - #{$navbar-height-xs});
@@ -300,23 +290,9 @@ div[class^='pipeline-section__'],
   }
 }
 
-// STATES
-
-@mixin selected {
-  .badge {
-    @include bg-gradient;
-    opacity: 1;
-
-    .fa-caret-right {
-      transform: rotate(90deg);
-    }
-  }
-}
-
 // INPUT
 
 .pipeline--input {
-
   .pipeline-tabs {
     left: 100vw;
   }
@@ -325,46 +301,45 @@ div[class^='pipeline-section__'],
     width: 100%;
   }
 
-  div.pipeline-nav-item__input {
+  div.pipeline-nav-item--input {
     @include selected;
-    width: calc(100% - (#{$entityTypes-nav-width} + #{$mapping-nav-width}));
+    width: calc(100% - (#{$entities-nav-width} + #{$mapping-nav-width}));
     color: $text-color;
   }
 
-  div.pipeline-nav-item__contracts {
+  div.pipeline-nav-item--contracts {
     display: block;
   }
 
-  div.pipeline-nav-item__entityTypes,
-  div.pipeline-nav-item__mapping,
-  div.pipeline-nav-item__output {
+  div.pipeline-nav-item--entities,
+  div.pipeline-nav-item--mapping,
+  div.pipeline-nav-item--output {
     width: 0;
   }
 
-  div.pipeline-section__input {
+  div.pipeline-section--input {
     width: 100%;
   }
 }
 
 // ENTITY TYPES
 
-.pipeline--entityTypes {
-
-  div.pipeline-nav-item__input {
+.pipeline--entities {
+  div.pipeline-nav-item--input {
     width: $column-width;
   }
 
-  div.pipeline-section__input {
+  div.pipeline-section--input {
     width: $column-width;
   }
 
-  div.pipeline-nav-item__entityTypes {
+  div.pipeline-nav-item--entities {
     @include selected;
     width: calc(100% - #{$column-width} - (#{$mapping-nav-width}));
     color: $white;
   }
 
-  div.pipeline-section__entityTypes {
+  div.pipeline-section--entities {
     width: calc(100% - #{$column-width});
 
     .fullscreen-toggle {
@@ -373,22 +348,21 @@ div[class^='pipeline-section__'],
   }
 }
 
-.pipeline--entityTypes .fullscreen {
-
-  div.pipeline-nav-item__input,
-  div.pipeline-nav-item__mapping {
+.pipeline--entities .fullscreen {
+  div.pipeline-nav-item--input,
+  div.pipeline-nav-item--mapping {
     width: 0;
   }
 
-  div.pipeline-section__input {
+  div.pipeline-section--input {
     width: 0;
   }
 
-  div.pipeline-nav-item__entityTypes {
+  div.pipeline-nav-item--entities {
     width: 100%;
   }
 
-  div.pipeline-section__entityTypes {
+  div.pipeline-section--entities {
     width: 100%;
   }
 }
@@ -396,30 +370,29 @@ div[class^='pipeline-section__'],
 // MAPPING
 
 .pipeline--mapping {
-
-  div.pipeline-nav-item__input {
+  div.pipeline-nav-item--input {
     width: $column-width;
   }
 
-  div.pipeline-section__input {
+  div.pipeline-section--input {
     width: $column-width;
   }
 
-  div.pipeline-nav-item__entityTypes {
+  div.pipeline-nav-item--entities {
     width: $column-width;
   }
 
-  div.pipeline-section__entityTypes {
+  div.pipeline-section--entities {
     width: $column-width;
   }
 
-  div.pipeline-nav-item__mapping {
+  div.pipeline-nav-item--mapping {
     @include selected;
     width: calc(100% - #{$column-width}*2);
     color: $white;
   }
 
-  div.pipeline-section__mapping {
+  div.pipeline-section--mapping {
     width: calc(100% - #{$column-width}*2);
 
     .fullscreen-toggle {
@@ -429,36 +402,33 @@ div[class^='pipeline-section__'],
 }
 
 .pipeline--mapping .fullscreen {
-
-  div.pipeline-nav-item__input,
-  div.pipeline-nav-item__entityTypes {
+  div.pipeline-nav-item--input,
+  div.pipeline-nav-item--entities {
     width: 0;
   }
 
-  div.pipeline-section__input,
-  div.pipeline-section__entityTypes {
+  div.pipeline-section--input,
+  div.pipeline-section--entities {
     width: 0;
   }
 
-  div.pipeline-nav-item__mapping {
+  div.pipeline-nav-item--mapping {
     width: 100%;
   }
 
-  div.pipeline-section__mapping {
+  div.pipeline-section--mapping {
     width: 100%;
   }
 }
 
-
 // OUTPUT
 
 .show-output {
-
   .pipeline-nav-items {
     width: calc(100vw - #{$output-width});
   }
 
-  div.pipeline-nav-item__output {
+  div.pipeline-nav-item--output {
     @include selected;
     width: $output-width;
   }
diff --git a/aether-ui/aether/ui/assets/css/_pipelines.scss b/aether-ui/aether/ui/assets/css/_pipelines.scss
index 84d77a019..349a00167 100644
--- a/aether-ui/aether/ui/assets/css/_pipelines.scss
+++ b/aether-ui/aether/ui/assets/css/_pipelines.scss
@@ -262,21 +262,6 @@
   }
 }
 
-@keyframes show-form {
-
-  0% {
-    max-height: 0;
-    padding: 0;
-    opacity: 0;
-  }
-
-  100% {
-    max-height: 200px;
-    padding: 2rem 0 3rem;
-    opacity: 1;
-  }
-}
-
 .pipeline-form {
   display: flex;
   align-items: center;
@@ -305,15 +290,8 @@
     width: 100%;
   }
 
-  @-webkit-keyframes autofill {
-    to {
-      color: $white;
-      background: rgba($text-color, .1);
-    }
-}
-
   .text-input:-webkit-autofill {
-    -webkit-animation-name: autofill;
+    -webkit-animation-name: autofill-dark;
     -webkit-animation-fill-mode: both;
   }
 
@@ -332,7 +310,6 @@
 }
 
 @media screen and (max-width: 1000px) {
-
   .pipelines {
     padding: 3rem 5vw;
   }
@@ -373,5 +350,4 @@
       margin-left: 1.5rem;
     }
   }
-
 }
diff --git a/aether-ui/aether/ui/assets/css/_section-entity-types.scss b/aether-ui/aether/ui/assets/css/_section-entity-types.scss
index 9536ece8c..d9c35e1e1 100644
--- a/aether-ui/aether/ui/assets/css/_section-entity-types.scss
+++ b/aether-ui/aether/ui/assets/css/_section-entity-types.scss
@@ -18,7 +18,7 @@
  * under the License.
  */
 
-.pipeline-section__entityTypes {
+.pipeline-section--entities {
   background: lighten($dark-blue, 5);
   color: $white;
 
@@ -123,29 +123,13 @@
   }
 }
 
-
-@keyframes show {
-  0% {
-    opacity: 0;
-  }
-
-  50% {
-    opacity: 0;
-  }
-
-  100% {
-    opacity: 1;
-  }
-}
-
-.pipeline--entityTypes .pipeline-section__entityTypes .section-right {
+.pipeline--entities .pipeline-section--entities .section-right {
   display: block;
   animation: show .5s;
 }
 
 @media screen and (min-width: 1200px) {
-
-  .pipeline-section__entityTypes .section-left {
+  .pipeline-section--entities .section-left {
     padding: $unit $unit-xl;
   }
 }
diff --git a/aether-ui/aether/ui/assets/css/_section-input.scss b/aether-ui/aether/ui/assets/css/_section-input.scss
index e0e334ce6..4970b4c2f 100644
--- a/aether-ui/aether/ui/assets/css/_section-input.scss
+++ b/aether-ui/aether/ui/assets/css/_section-input.scss
@@ -18,8 +18,7 @@
  * under the License.
  */
 
-.pipeline-section__input {
-
+.pipeline-section--input {
   .section-body {
     display: flex;
     padding: 2rem 0 0;
@@ -135,7 +134,6 @@
   }
 
   .group .group {
-
     .field,
     .group-title {
       padding-top: .1rem;
@@ -152,21 +150,18 @@
   }
 
   .group .group .group {
-
     .field,
     .group-title {
       padding-left: $indent;
     }
 
     .group-list:not(:only-child) {
-
       .field,
       .group-title {
         padding-left: $indent * 2;
       }
 
       .group-list:not(:only-child) {
-
         .field,
         .group-title {
           padding-left: $indent * 3;
@@ -176,28 +171,13 @@
   }
 }
 
-@keyframes show {
-  0% {
-    opacity: 0;
-  }
-
-  50% {
-    opacity: 0;
-  }
-
-  100% {
-    opacity: 1;
-  }
-}
-
-.pipeline--input .pipeline-section__input .section-right {
+.pipeline--input .pipeline-section--input .section-right {
   display: flex;
   animation: show 1s;
 }
 
 @media screen and (min-width: 1200px) {
-
-  .pipeline-section__input .section-left {
+  .pipeline-section--input .section-left {
     padding: $unit $unit-xl;
   }
 }
diff --git a/aether-ui/aether/ui/assets/css/_section-mapping.scss b/aether-ui/aether/ui/assets/css/_section-mapping.scss
index 6ef311811..d788c5340 100644
--- a/aether-ui/aether/ui/assets/css/_section-mapping.scss
+++ b/aether-ui/aether/ui/assets/css/_section-mapping.scss
@@ -18,7 +18,7 @@
  * under the License.
  */
 
-.pipeline-section__mapping {
+.pipeline-section--mapping {
   background: $dark-blue;
   color: $white;
 
@@ -29,7 +29,6 @@
   }
 
   .tabs {
-
     .tab.selected {
       box-shadow: 1px 1px 1px rgba($black, .2) inset;
       background-color: darken($dark-blue, 3);
@@ -171,21 +170,7 @@
   }
 }
 
-@keyframes show {
-  0% {
-    opacity: 0;
-  }
-
-  50% {
-    opacity: 0;
-  }
-
-  100% {
-    opacity: 1;
-  }
-}
-
-.pipeline--mapping .pipeline-section__mapping .section-body {
+.pipeline--mapping .pipeline-section--mapping .section-body {
   display: flex;
   animation: show 1s;
 }
diff --git a/aether-ui/aether/ui/assets/css/base/_global.scss b/aether-ui/aether/ui/assets/css/base/_global.scss
index 2efb4653f..564245c9e 100644
--- a/aether-ui/aether/ui/assets/css/base/_global.scss
+++ b/aether-ui/aether/ui/assets/css/base/_global.scss
@@ -172,17 +172,10 @@ textarea:active {
   outline: 0;
 }
 
-@-webkit-keyframes autofill {
-  to {
-    color: $white;
-    background: transparent;
-  }
-}
-
 input:-webkit-autofill,
 textarea:-webkit-autofill,
 select:-webkit-autofill {
-  -webkit-animation-name: autofill;
+  -webkit-animation-name: autofill-transparent;
   -webkit-animation-fill-mode: both;
 }
 
@@ -201,7 +194,7 @@ select:-webkit-autofill {
   }
 }
 
-.textarea-header+textarea {
+.textarea-header + textarea {
   border-radius: 0 0 $border-radius $border-radius;
 }
 
@@ -367,7 +360,6 @@ code {
   }
 
   input:checked + label {
-
     &::before {
       background: $action-color;
     }
@@ -430,7 +422,6 @@ code {
   @extend .check-default;
 
   input:checked + label {
-
     cursor: default;
 
     &::before {
@@ -458,9 +449,7 @@ code {
   }
 }
 
-
 @media screen and (max-width: 768px) {
-
   body,
   html {
     font-size: 13px;
diff --git a/aether-ui/aether/ui/assets/apps/components/Portal.jsx b/aether-ui/aether/ui/assets/css/base/_keyframes.scss
similarity index 50%
rename from aether-ui/aether/ui/assets/apps/components/Portal.jsx
rename to aether-ui/aether/ui/assets/css/base/_keyframes.scss
index 13cf4b757..4b4f84787 100644
--- a/aether-ui/aether/ui/assets/apps/components/Portal.jsx
+++ b/aether-ui/aether/ui/assets/css/base/_keyframes.scss
@@ -18,34 +18,46 @@
  * under the License.
  */
 
-import { useState, useEffect } from 'react'
-import ReactDOM from 'react-dom'
-
-// https://reactjs.org/docs/portals.html
-
-const Portal = ({ children, onEscape, onEnter }) => {
-  const [element] = useState(document.createElement('div'))
+@keyframes fade-in {
+  0%   { opacity: 0; }
+  100% { opacity: 1; }
+}
 
-  useEffect(() => {
-    const onKeyDown = (event) => {
-      if (event.key === 'Escape') {
-        onEscape && onEscape(event)
-      }
-      if (event.key === 'Enter') {
-        onEnter && onEnter(event)
-      }
-    }
+@keyframes show {
+  0%   { opacity: 0; }
+  50%  { opacity: 0; }
+  100% { opacity: 1; }
+}
 
-    document.body.appendChild(element)
-    document.addEventListener('keydown', onKeyDown)
+@keyframes show-form {
+  0% {
+    max-height: 0;
+    padding: 0;
+    opacity: 0;
+  }
+
+  100% {
+    max-height: 200px;
+    padding: 2rem 0 3rem;
+    opacity: 1;
+  }
+}
 
-    return () => {
-      document.body.removeChild(element)
-      document.removeEventListener('keydown', onKeyDown)
-    }
-  })
+@keyframes slide-up {
+  0%   { top: 100vh; }
+  100% { top: $navbar-height-xs; }
+}
 
-  return ReactDOM.createPortal(children, element)
+@-webkit-keyframes autofill-transparent {
+  to {
+    color: $white;
+    background: transparent;
+  }
 }
 
-export default Portal
+@-webkit-keyframes autofill-dark {
+  to {
+    color: $white;
+    background: rgba($text-color, .1);
+  }
+}
diff --git a/aether-ui/aether/ui/assets/css/base/_messages.scss b/aether-ui/aether/ui/assets/css/base/_messages.scss
index b82dca024..878acd6fb 100644
--- a/aether-ui/aether/ui/assets/css/base/_messages.scss
+++ b/aether-ui/aether/ui/assets/css/base/_messages.scss
@@ -33,6 +33,6 @@
   color: $red;
 }
 
-form .error-message+textarea {
+form .error-message + textarea {
   background: rgba($red, .1);
 }
diff --git a/aether-ui/aether/ui/assets/css/base/_mixins.scss b/aether-ui/aether/ui/assets/css/base/_mixins.scss
index 61975cbf0..fcb8a2174 100644
--- a/aether-ui/aether/ui/assets/css/base/_mixins.scss
+++ b/aether-ui/aether/ui/assets/css/base/_mixins.scss
@@ -77,3 +77,16 @@
 @mixin shadow-emboss-color {
   box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($text-color, .3) inset, 0 0 4px rgba($text-color, .1);
 }
+
+// STATES
+
+@mixin selected {
+  .badge {
+    @include bg-gradient;
+    opacity: 1;
+
+    .fa-caret-right {
+      transform: rotate(90deg);
+    }
+  }
+}
diff --git a/aether-ui/aether/ui/assets/css/base/_modal.scss b/aether-ui/aether/ui/assets/css/base/_modal.scss
index 6c86784d7..055c3dae0 100644
--- a/aether-ui/aether/ui/assets/css/base/_modal.scss
+++ b/aether-ui/aether/ui/assets/css/base/_modal.scss
@@ -100,7 +100,6 @@
     margin-top: $unit;
 
     .btn {
-
       &:not(:first-child) {
         margin-left: $unit;
       }
@@ -121,7 +120,6 @@
 }
 
 @media screen and (min-width: 576px) {
-
   .modal-dialog {
     width: 70vw;
     max-width: 800px;
diff --git a/aether-ui/aether/ui/assets/css/base/_nav-bar.scss b/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
index fb2339084..b3eef8ceb 100644
--- a/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
+++ b/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
@@ -29,11 +29,6 @@
   top: 72px;
 }
 
-@keyframes fade-in {
-  0%   { opacity: 0; }
-  100% { opacity: 1; }
-}
-
 .top-nav-breadcrumb {
   animation: fade-in .6s;
   letter-spacing: .06em;
@@ -103,7 +98,6 @@
   }
 
   .top-nav:hover {
-
     .breadcrumb-links a,
     .top-nav-logo,
     .status-publish {
@@ -134,7 +128,6 @@
 }
 
 .pipeline--input {
-
   .breadcrumb-links a {
     opacity: 1;
     max-width: 100px;
diff --git a/aether-ui/aether/ui/assets/css/base/_settings.scss b/aether-ui/aether/ui/assets/css/base/_settings.scss
index 102a22d6e..8be18dba8 100644
--- a/aether-ui/aether/ui/assets/css/base/_settings.scss
+++ b/aether-ui/aether/ui/assets/css/base/_settings.scss
@@ -49,9 +49,9 @@ $hover-color:        $purple;
 // Fonts
 // ************************************
 
-$body-font-family: 'Open Sans', Arial, sans-serif;
-$code-font-family: 'Fira Mono', SFMono-Regular, Menlo, Monaco, monospace;
-$icon-font:        'Font Awesome 5 Free';
+$body-font-family:   'Open Sans', Arial, sans-serif;
+$code-font-family:   'Fira Mono', SFMono-Regular, Menlo, Monaco, monospace;
+$icon-font:          'Font Awesome 5 Free';
 
 $font-size-xxs:      .7rem;
 $font-size-xs:       .86rem;
@@ -66,12 +66,14 @@ $font-size-xxl:      3.2rem;
 // Sizes
 // ************************************
 
-$unit: 1.2rem;
-$unit-xl: 2rem;
-$border-radius: 4px;
+$unit:               1.2rem;
+$unit-xl:            2rem;
+$border-radius:      4px;
 
-$column-width: 20vw;
-$output-width: 25vw;
-$navbar-height-xs: 2.3rem;
+$column-width:       20vw;
+$output-width:       25vw;
+$navbar-height-xs:   2.3rem;
 
-$indent: 1.2rem;
+$indent:             1.2rem;
+
+$transition-speed:   .6s;
diff --git a/aether-ui/aether/ui/assets/css/base/_styleguide.scss b/aether-ui/aether/ui/assets/css/base/_styleguide.scss
index 683671048..f73838d73 100644
--- a/aether-ui/aether/ui/assets/css/base/_styleguide.scss
+++ b/aether-ui/aether/ui/assets/css/base/_styleguide.scss
@@ -123,7 +123,6 @@ $colors: (
     $i: index($colors-list, $current-color);
 
     .entity-color-sample:nth-child(#{$i}) {
-
       div {
         background-color: $current-color;
         height: 80px;
diff --git a/aether-ui/aether/ui/assets/css/index.scss b/aether-ui/aether/ui/assets/css/index.scss
index 60d3f364c..7fa1a36ac 100644
--- a/aether-ui/aether/ui/assets/css/index.scss
+++ b/aether-ui/aether/ui/assets/css/index.scss
@@ -33,15 +33,18 @@
 @import 'base/settings';
 @import 'base/color-codes';
 @import 'base/mixins';
+@import 'base/keyframes';
 @import 'base/global';
 @import 'base/buttons';
 @import 'base/nav-bar';
 @import 'base/loading-spinner';
 @import 'base/modal';
 @import 'base/messages';
+
 @import 'pipelines';
 @import 'pipeline';
 @import 'pipeline-settings';
+
 @import 'section-input';
 @import 'section-entity-types';
 @import 'section-mapping';
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index d7428b95f..f7b6e2a91 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -35,6 +35,7 @@
     "react-router-dom": "~5.1.0",
     "redux": "~4.0.0",
     "redux-thunk": "~2.3.0",
+    "uuid": "~7.0.0",
     "webpack-google-cloud-storage-plugin": "~0.9.0",
     "webpack-s3-plugin": "~1.0.3",
     "whatwg-fetch": "~3.0.0"
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index dcb7c206d..1144ec4c0 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -78,7 +78,6 @@ services:
 
       KEYCLOAK_USER: ${KEYCLOAK_ADMIN_USERNAME}
       KEYCLOAK_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
-      KEYCLOAK_HTTP_PORT: 8080
 
 
   # ---------------------------------

From 5d87e8f9f1ac21d55524cb913da0f3b42a4a9a97 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Fri, 29 May 2020 12:26:04 +0200
Subject: [PATCH 06/29] chore: upgrade dependencies (#836)

* chore: upgrade python dependencies

* chore(ui): upgrade node dependencies

* chore(ui): replace sass-lint with stylelint

* fix: flake8

* chore: more upgrades

* fix: useless noqa

* chore: more upgrades

* chore: more upgrades

* fix: merge

* chore: missing update uuid

* fix: duplicated entry
---
 .../aether/client/__init__.py                 |   4 +-
 aether-kernel/conf/pip/requirements.txt       |  39 +++--
 aether-odk-module/conf/pip/requirements.txt   |  39 ++---
 aether-producer/conf/pip/requirements.txt     |  25 ++--
 aether-producer/producer/db.py                |  13 +-
 aether-producer/producer/kernel_db.py         |  10 +-
 aether-ui/aether/ui/api/tests/test_models.py  |   2 +-
 aether-ui/aether/ui/assets/conf/sass-lint.yml |  59 --------
 .../ui/assets/css/_pipeline-settings.scss     |  10 +-
 aether-ui/aether/ui/assets/css/_pipeline.scss |  49 +++---
 .../aether/ui/assets/css/_pipelines.scss      |  63 ++++----
 .../ui/assets/css/_section-entity-types.scss  |  23 +--
 .../aether/ui/assets/css/_section-input.scss  |  18 +--
 .../ui/assets/css/_section-mapping.scss       |  36 +++--
 .../aether/ui/assets/css/_section-output.scss |   4 +-
 .../aether/ui/assets/css/base/_buttons.scss   |  43 ++++--
 .../ui/assets/css/base/_color-codes.scss      |  20 +--
 .../aether/ui/assets/css/base/_fonts.scss     |   5 -
 .../aether/ui/assets/css/base/_global.scss    | 141 +++++++++---------
 .../aether/ui/assets/css/base/_keyframes.scss |   6 +-
 .../ui/assets/css/base/_loading-spinner.scss  |  13 +-
 .../aether/ui/assets/css/base/_messages.scss  |   4 +-
 .../aether/ui/assets/css/base/_mixins.scss    |  54 +++++--
 .../aether/ui/assets/css/base/_modal.scss     |  13 +-
 .../aether/ui/assets/css/base/_nav-bar.scss   |  19 +--
 .../aether/ui/assets/css/base/_settings.scss  |  14 +-
 .../ui/assets/css/base/_styleguide.scss       |  12 +-
 aether-ui/aether/ui/assets/css/index.scss     |   9 +-
 aether-ui/aether/ui/assets/package.json       |  36 +++--
 aether-ui/conf/pip/requirements.txt           |  37 ++---
 30 files changed, 422 insertions(+), 398 deletions(-)
 delete mode 100644 aether-ui/aether/ui/assets/conf/sass-lint.yml

diff --git a/aether-client-library/aether/client/__init__.py b/aether-client-library/aether/client/__init__.py
index c205a1af3..5210b1901 100644
--- a/aether-client-library/aether/client/__init__.py
+++ b/aether-client-library/aether/client/__init__.py
@@ -40,8 +40,8 @@
 # monkey patch so that bulk insertion works
 from .patches import patched__marshal_object, patched__unmarshal_object
 
-bravado_core.marshal._marshal_object = patched__marshal_object        # noqa
-bravado_core.unmarshal._unmarshal_object = patched__unmarshal_object  # noqa
+bravado_core.marshal._marshal_object = patched__marshal_object
+bravado_core.unmarshal._unmarshal_object = patched__unmarshal_object
 
 _SPEC_URL = '{}/v1/schema/?format=openapi'
 
diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 384fc31bd..2c9f27524 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -16,8 +16,8 @@ aether.python==1.0.17
 aether.sdk==1.2.22
 attrs==19.3.0
 autopep8==1.5.2
-boto3==1.12.48
-botocore==1.15.48
+boto3==1.13.19
+botocore==1.16.19
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -30,15 +30,15 @@ cryptography==2.9.2
 decorator==4.4.2
 Django==2.2.12
 django-autofixture==0.12.1
-django-cacheops==4.2
+django-cacheops==5.0
 django-cleanup==4.0.0
-django-cors-headers==3.2.1
+django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-filter==2.2.0
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
 django-prometheus==2.0.0
-django-redis==4.11.0
+django-redis==4.12.1
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
@@ -47,15 +47,14 @@ docutils==0.15.2
 drf-dynamic-fields==0.3.1
 drf-yasg==1.17.1
 eha-jsonpath==0.5.1
-entrypoints==0.3
 et-xmlfile==1.0.1
-flake8==3.7.9
-flake8-quotes==3.0.0
+flake8==3.8.2
+flake8-quotes==3.2.0
 funcy==1.14
 google-api-core==1.17.0
-google-auth==1.14.1
+google-auth==1.16.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.0
+google-cloud-storage==1.28.1
 google-resumable-media==0.5.0
 googleapis-common-protos==1.51.0
 gprof2dot==2019.11.30
@@ -65,24 +64,24 @@ inflection==0.4.0
 itypes==1.2.0
 jdcal==1.4.1
 Jinja2==2.11.2
-jmespath==0.9.5
+jmespath==0.10.0
 jsonpath-ng==1.5.1
 jsonschema==3.2.0
-lxml==4.5.0
+lxml==4.5.1
 MarkupSafe==1.1.1
 mccabe==0.6.1
 minio==5.0.10
 openpyxl==3.0.3
-packaging==20.3
+packaging==20.4
 ply==3.11
-prometheus-client==0.7.1
-protobuf==3.11.3
+prometheus-client==0.8.0
+protobuf==3.12.2
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.5.0
+pycodestyle==2.6.0
 pycparser==2.20
-pyflakes==2.1.1
+pyflakes==2.2.0
 Pygments==2.6.1
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
@@ -90,14 +89,14 @@ pyrsistent==0.16.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
-redis==3.4.1
+redis==3.5.2
 requests==2.23.0
 rsa==4.0
 ruamel.yaml==0.16.10
 ruamel.yaml.clib==0.2.0
 s3transfer==0.3.3
-sentry-sdk==0.14.3
-six==1.14.0
+sentry-sdk==0.14.4
+six==1.15.0
 spavro==1.1.23
 sqlparse==0.3.1
 tblib==1.6.0
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index 61e032808..af37d31be 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -14,8 +14,8 @@
 
 aether.sdk==1.2.22
 autopep8==1.5.2
-boto3==1.12.48
-botocore==1.15.48
+boto3==1.13.19
+botocore==1.16.19
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -24,59 +24,59 @@ configparser==5.0.0
 coverage==5.1
 cryptography==2.9.2
 Django==2.2.12
-django-cacheops==4.2
+django-cacheops==5.0
 django-cleanup==4.0.0
-django-cors-headers==3.2.1
+django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-prometheus==2.0.0
-django-redis==4.11.0
+django-redis==4.12.1
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
 djangorestframework==3.11.0
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
-entrypoints==0.3
-flake8==3.7.9
-flake8-quotes==3.0.0
+flake8==3.8.2
+flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
 google-api-core==1.17.0
-google-auth==1.14.1
+google-auth==1.16.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.0
+google-cloud-storage==1.28.1
 google-resumable-media==0.5.0
 googleapis-common-protos==1.51.0
 gprof2dot==2019.11.30
 idna==2.9
+importlib-metadata==1.6.0
 Jinja2==2.11.2
-jmespath==0.9.5
+jmespath==0.10.0
 linecache2==1.0.0
-lxml==4.5.0
+lxml==4.5.1
 MarkupSafe==1.1.1
 mccabe==0.6.1
 minio==5.0.10
-prometheus-client==0.7.1
-protobuf==3.11.3
+prometheus-client==0.8.0
+protobuf==3.12.2
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.5.0
+pycodestyle==2.6.0
 pycparser==2.20
-pyflakes==2.1.1
+pyflakes==2.2.0
 Pygments==2.6.1
 pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
 pyxform==1.1.0
-redis==3.4.1
+redis==3.5.2
 requests==2.23.0
 rsa==4.0
 s3transfer==0.3.3
-sentry-sdk==0.14.3
-six==1.14.0
+sentry-sdk==0.14.4
+six==1.15.0
 spavro==1.1.23
 sqlparse==0.3.1
 tblib==1.6.0
@@ -86,3 +86,4 @@ unittest2==1.1.0
 urllib3==1.25.9
 uWSGI==2.0.18
 xlrd==1.2.0
+zipp==3.1.0
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index 74845825b..9c58029ea 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -17,13 +17,12 @@ certifi==2020.4.5.1
 cffi==1.14.0
 chardet==3.0.4
 click==7.1.2
-confluent-kafka==1.4.1
+confluent-kafka==1.4.2
 cryptography==2.9.2
-entrypoints==0.3
-flake8==3.7.9
-flake8-quotes==3.0.0
+flake8==3.8.2
+flake8-quotes==3.2.0
 Flask==1.1.2
-gevent==20.4.0
+gevent==20.5.2
 greenlet==0.4.15
 idna==2.9
 importlib-metadata==1.6.0
@@ -31,23 +30,25 @@ itsdangerous==1.1.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
-more-itertools==8.2.0
-packaging==20.3
+more-itertools==8.3.0
+packaging==20.4
 pluggy==0.13.1
 psycogreen==1.0.2
 psycopg2-binary==2.8.5
 py==1.8.1
-pycodestyle==2.5.0
+pycodestyle==2.6.0
 pycparser==2.20
-pyflakes==2.1.1
+pyflakes==2.2.0
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
-pytest==5.4.1
+pytest==5.4.2
 requests==2.23.0
-six==1.14.0
+six==1.15.0
 spavro==1.1.23
-SQLAlchemy==1.3.16
+SQLAlchemy==1.3.17
 urllib3==1.25.9
 wcwidth==0.1.9
 Werkzeug==1.0.1
 zipp==3.1.0
+zope.event==4.4
+zope.interface==5.1.0
diff --git a/aether-producer/producer/db.py b/aether-producer/producer/db.py
index 12f6280b7..09230bce1 100644
--- a/aether-producer/producer/db.py
+++ b/aether-producer/producer/db.py
@@ -16,6 +16,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# flake8: noqa: E402
+
+# need to patch sockets to make requests async
+from gevent import monkey
+monkey.patch_all()
+import psycogreen.gevent
+psycogreen.gevent.patch_psycopg()
+
 from datetime import datetime
 import signal
 import sys
@@ -25,11 +33,6 @@
 from gevent.event import AsyncResult
 from gevent.queue import PriorityQueue, Queue
 
-# need to patch sockets to make requests async
-monkey.patch_all()  # noqa
-import psycogreen.gevent
-psycogreen.gevent.patch_psycopg()  # noqa
-
 import psycopg2
 from psycopg2 import sql
 from psycopg2.extras import DictCursor
diff --git a/aether-producer/producer/kernel_db.py b/aether-producer/producer/kernel_db.py
index eb96d218f..2f6676532 100644
--- a/aether-producer/producer/kernel_db.py
+++ b/aether-producer/producer/kernel_db.py
@@ -16,13 +16,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from datetime import datetime
+# flake8: noqa: E402
 
-from gevent import monkey
 # need to patch sockets to make requests async
-monkey.patch_all()  # noqa
+from gevent import monkey
+monkey.patch_all()
 import psycogreen.gevent
-psycogreen.gevent.patch_psycopg()  # noqa
+psycogreen.gevent.patch_psycopg()
+
+from datetime import datetime
 
 import psycopg2
 from psycopg2 import sql
diff --git a/aether-ui/aether/ui/api/tests/test_models.py b/aether-ui/aether/ui/api/tests/test_models.py
index 3c1fd3cbf..8e8628ee6 100644
--- a/aether-ui/aether/ui/api/tests/test_models.py
+++ b/aether-ui/aether/ui/api/tests/test_models.py
@@ -182,7 +182,7 @@ def test__contract__save__with_server_error(self, mock_post):
         )
         self.assertEqual(
             contract.mapping_errors,
-            [{'description': f'It was not possible to validate the contract: Internal Server Error'}]
+            [{'description': 'It was not possible to validate the contract: Internal Server Error'}]
         )
         self.assertEqual(contract.output, [])
         mock_post.assert_called_once()
diff --git a/aether-ui/aether/ui/assets/conf/sass-lint.yml b/aether-ui/aether/ui/assets/conf/sass-lint.yml
deleted file mode 100644
index ad36412b7..000000000
--- a/aether-ui/aether/ui/assets/conf/sass-lint.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-options:
-  max-warnings: 300
-
-files:
-  include:
-    - './css/**/*.scss'
-
-rules:
-
-  # ========================
-  # possible values:
-  #
-  # 0  --  disabled
-  # 1  --  warning
-  # 2  --  error
-  # ========================
-
-  indentation:
-    - 2
-    - size: 2
-
-  quotes:
-    - 2
-    - style: single       # values: single | double
-
-  border-zero:
-    - 2
-    - convention: 0       # values: 0 | none
-
-  hex-notation:
-    - 2
-    - style: lowercase    # values: lowercase | uppercase
-
-  nesting-depth:
-    - 1
-    - max-depth: 3
-
-
-  placeholder-in-extend:      0
-  property-sort-order:        0
-  space-after-colon:          2
-
-  no-color-literals:          0
-  no-css-comments:            0
-  no-duplicate-properties:    2
-  no-important:               1
-  no-qualifying-elements:     0
-  no-trailing-whitespace:     2
-  no-transition-all:          0
-  no-vendor-prefixes:         0
-
-  force-attribute-nesting:    0
-  force-element-nesting:      0
-  force-pseudo-nesting:       0
-
-  class-name-format:          2
-  variable-name-format:       2
-
-  # see more: https://github.com/sasstools/sass-lint/tree/develop/docs/rules
diff --git a/aether-ui/aether/ui/assets/css/_pipeline-settings.scss b/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
index a5fc0a68b..729e6235d 100644
--- a/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
+++ b/aether-ui/aether/ui/assets/css/_pipeline-settings.scss
@@ -27,8 +27,8 @@
   transition: left $transition-speed;
   right: 0;
   z-index: 4;
-  animation: show .5s;
-  padding: .6rem 1rem;
+  animation: show 0.5s;
+  padding: 0.6rem 1rem;
   overflow: auto;
 
   .status-publish {
@@ -63,7 +63,7 @@ div.code {
   margin-top: 2rem;
   padding: 1.6rem 2rem 1.2rem 4rem;
   border-radius: $border-radius * 4;
-  background-color: rgba($light-grey, .1);
+  background-color: rgba($light-grey, 0.1);
 
   .toggle-default {
     margin-left: -2.4rem;
@@ -83,7 +83,7 @@ div.code {
 .settings-section {
   margin: 2rem 0;
   padding: 2rem 0;
-  border-top: 1px dashed rgba($light-grey, .4);
+  border-top: 1px dashed rgba($light-grey, 0.4);
 }
 
 .settings-actions {
@@ -96,7 +96,7 @@ div.code {
 
 .clipboard {
   font-size: 1rem;
-  margin: .2rem;
+  margin: 0.2rem;
   color: lighten($dark-blue, 5);
   cursor: pointer;
 }
diff --git a/aether-ui/aether/ui/assets/css/_pipeline.scss b/aether-ui/aether/ui/assets/css/_pipeline.scss
index e6413a92d..725c0ab71 100644
--- a/aether-ui/aether/ui/assets/css/_pipeline.scss
+++ b/aether-ui/aether/ui/assets/css/_pipeline.scss
@@ -26,14 +26,14 @@ $output-nav-width:      10rem;
 
 .pipeline {
   background: $white;
-  box-shadow: 0 -1px 3px rgba($text-color, .3);
+  box-shadow: 0 -1px 3px rgba($text-color, 0.3);
   position: fixed;
   display: flex;
   top: $navbar-height-xs;
   bottom: 0;
   width: 100vw;
-  transition: top .4s .5s;
-  animation: slide-up .6s;
+  transition: top 0.4s 0.5s;
+  animation: slide-up 0.6s;
   z-index: 4;
 }
 
@@ -48,13 +48,13 @@ $output-nav-width:      10rem;
 }
 
 .pipeline-tab {
-  padding: .4rem .8rem;
+  padding: 0.4rem 0.8rem;
   display: flex;
   align-items: center;
   height: $navbar-height-xs;
   cursor: pointer;
   min-width: 0;
-  border-left: 1px solid rgba($white, .1);
+  border-left: 1px solid rgba($white, 0.1);
 
   .contract-name {
     white-space: nowrap;
@@ -79,7 +79,7 @@ $output-nav-width:      10rem;
 
   &.active {
     background: lighten($dark-blue, 5);
-    border-bottom: 1px solid rgba($white, .1);
+    border-bottom: 1px solid rgba($white, 0.1);
     border-left: 0;
 
     .settings-button {
@@ -92,15 +92,15 @@ $output-nav-width:      10rem;
   }
 
   &:last-of-type {
-    margin-right: .3rem;
+    margin-right: 0.3rem;
   }
 }
 
 .new-contract {
   margin-left: auto;
-  transition: opacity .4s;
+  transition: opacity 0.4s;
   align-self: center;
-  margin-right: .3rem;
+  margin-right: 0.3rem;
   flex: 0 0 auto;
 }
 
@@ -125,8 +125,9 @@ $output-nav-width:      10rem;
   &::before,
   &::after {
     @include shadow-flat-dark;
+
     font: 1rem $icon-font;
-    padding: .5rem;
+    padding: 0.5rem;
     position: absolute;
     top: 0;
     width: 2rem;
@@ -150,14 +151,14 @@ $output-nav-width:      10rem;
   span {
     padding: 0 2.3rem;
     opacity: 0;
-    transition: opacity .2s;
+    transition: opacity 0.2s;
   }
 
   &:focus,
   &:hover {
     &::before,
     &::after {
-      background-color: rgba($action-color-b, .2);
+      background-color: rgba($action-color-b, 0.2);
     }
 
     span {
@@ -201,9 +202,10 @@ $output-nav-width:      10rem;
 
 div[class^='pipeline-nav-item--'] {
   @include label;
+
   color: $action-color-b;
   font-weight: 600;
-  padding: .6rem 0;
+  padding: 0.6rem 0;
   transition: all $transition-speed;
   cursor: pointer;
   overflow: hidden;
@@ -220,14 +222,15 @@ div[class^='pipeline-nav-item--'] {
     width: 1.8rem;
     min-width: 1.8rem;
     height: 1.8rem;
-    padding: .4rem 0;
-    opacity: .4;
-    margin: 0 .8rem;
+    padding: 0.4rem 0;
+    opacity: 0.4;
+    margin: 0 0.8rem;
   }
 
   &:hover {
     .badge {
       @include bg-gradient;
+
       opacity: 1;
     }
   }
@@ -274,7 +277,7 @@ div[class^='pipeline-section--'],
   width: 0;
   height: calc(100vh - #{$navbar-height-xs});
   padding-top: $pipeline-nav-height;
-  transition: width $transition-speed, height .4s .5s;
+  transition: width $transition-speed, height 0.4s 0.5s;
   position: relative;
 }
 
@@ -303,6 +306,7 @@ div[class^='pipeline-section--'],
 
   div.pipeline-nav-item--input {
     @include selected;
+
     width: calc(100% - (#{$entities-nav-width} + #{$mapping-nav-width}));
     color: $text-color;
   }
@@ -335,6 +339,7 @@ div[class^='pipeline-section--'],
 
   div.pipeline-nav-item--entities {
     @include selected;
+
     width: calc(100% - #{$column-width} - (#{$mapping-nav-width}));
     color: $white;
   }
@@ -388,12 +393,13 @@ div[class^='pipeline-section--'],
 
   div.pipeline-nav-item--mapping {
     @include selected;
-    width: calc(100% - #{$column-width}*2);
+
+    width: calc(100% - #{$column-width} * 2);
     color: $white;
   }
 
   div.pipeline-section--mapping {
-    width: calc(100% - #{$column-width}*2);
+    width: calc(100% - #{$column-width} * 2);
 
     .fullscreen-toggle {
       display: flex;
@@ -430,6 +436,7 @@ div[class^='pipeline-section--'],
 
   div.pipeline-nav-item--output {
     @include selected;
+
     width: $output-width;
   }
 
@@ -438,7 +445,7 @@ div[class^='pipeline-section--'],
   }
 
   .pipeline-output {
-    width: calc(#{$output-width} + .2vw);
-    border-left: 1px solid rgba($black, .5);
+    width: calc(#{$output-width} + 0.2vw);
+    border-left: 1px solid rgba($black, 0.5);
   }
 }
diff --git a/aether-ui/aether/ui/assets/css/_pipelines.scss b/aether-ui/aether/ui/assets/css/_pipelines.scss
index 349a00167..408c9d82e 100644
--- a/aether-ui/aether/ui/assets/css/_pipelines.scss
+++ b/aether-ui/aether/ui/assets/css/_pipelines.scss
@@ -20,6 +20,7 @@
 
 .pipelines-container {
   @include bg-gradient;
+
   color: $white;
   min-height: 100vh;
 
@@ -32,9 +33,9 @@
     width: 110vw;
     height: 110vw;
     border-radius: 100%;
-    border: 1vw solid rgba($white, .1);
-    border-right: 6vw solid rgba($white, .1);
-    border-top: 4vw solid rgba($white, .1);
+    border: 1vw solid rgba($white, 0.1);
+    border-right: 6vw solid rgba($white, 0.1);
+    border-top: 4vw solid rgba($white, 0.1);
   }
 
   &::after {
@@ -46,9 +47,9 @@
     width: 70vw;
     height: 70vw;
     border-radius: 100%;
-    border: .5vw solid rgba($white, .1);
-    border-right: 5vw solid rgba($white, .1);
-    border-top: 3vw solid rgba($white, .1);
+    border: 0.5vw solid rgba($white, 0.1);
+    border-right: 5vw solid rgba($white, 0.1);
+    border-top: 3vw solid rgba($white, 0.1);
     z-index: 0;
   }
 
@@ -70,7 +71,7 @@
 }
 
 .pipeline-previews {
-  border-top: 1px dashed rgba($white, .2);
+  border-top: 1px dashed rgba($white, 0.2);
 }
 
 .pipeline-preview {
@@ -79,7 +80,7 @@
   position: relative;
   padding: 1rem;
   margin-top: 1.4rem;
-  background: rgba($background-color, .2);
+  background: rgba($background-color, 0.2);
   border-radius: $border-radius;
 }
 
@@ -102,9 +103,10 @@
 
 .preview-input {
   @include card;
-  margin-right: .8rem;
+
+  margin-right: 0.8rem;
   width: 25%;
-  transition: box-shadow .5s;
+  transition: box-shadow 0.5s;
 
   .badge {
     margin-left: 0;
@@ -112,8 +114,10 @@
 
   &:hover {
     cursor: pointer;
-    box-shadow: 3px 3px 5px rgba($text-color, .4), -1px -1px 1px $light-grey inset;
     background: $background-color;
+    box-shadow:
+      3px 3px 5px rgba($text-color, 0.4),
+      -1px -1px 1px $light-grey inset;
   }
 }
 
@@ -122,7 +126,7 @@
 
   .tag {
     position: relative;
-    top: -.2rem;
+    top: -0.2rem;
   }
 }
 
@@ -153,11 +157,12 @@
 
 .preview-contract {
   @include card;
-  background: rgba($mid-blue, .8);
+
+  background: rgba($mid-blue, 0.8);
   color: $white;
-  margin-bottom: .8rem;
+  margin-bottom: 0.8rem;
   position: relative;
-  transition: box-shadow .5s;
+  transition: box-shadow 0.5s;
 
   &:last-child {
     margin-bottom: 0;
@@ -172,25 +177,25 @@
     color: $action-color-b;
     font-weight: 300;
     font-size: $font-size-xl;
-    margin-bottom: .5rem;
-    transition: color .3s;
+    margin-bottom: 0.5rem;
+    transition: color 0.3s;
     word-wrap: break-word;
     flex: 1;
     min-width: 0;
   }
 
   &.pipeline-readonly {
-    @include stripy(rgba($mid-blue, .8), rgba($mid-blue, 1));
+    @include stripy(rgba($mid-blue, 0.8), rgba($mid-blue, 1));
 
     .tag {
       position: relative;
-      top: -.2rem;
+      top: -0.2rem;
     }
   }
 
   &:hover {
     cursor: pointer;
-    box-shadow: 3px 3px 5px rgba($text-color, .4);
+    box-shadow: 3px 3px 5px rgba($text-color, 0.4);
     background: rgba($mid-blue, 1);
 
     .contract-name {
@@ -233,7 +238,7 @@
 
 .contract-publish {
   padding-top: 1rem;
-  border-top: 1px dotted rgba($grey, .5);
+  border-top: 1px dotted rgba($grey, 0.5);
   width: 100%;
   display: flex;
   align-items: baseline;
@@ -266,7 +271,7 @@
   display: flex;
   align-items: center;
   padding: 2rem 0;
-  animation: show-form .5s;
+  animation: show-form 0.5s;
   overflow: hidden;
 
   .form-group {
@@ -279,12 +284,12 @@
     position: absolute;
     top: -2rem;
     opacity: 0;
-    transition: opacity .3s;
+    transition: opacity 0.3s;
   }
 
   .text-input {
     font-size: $font-size-xl;
-    background: rgba($text-color, .1);
+    background: rgba($text-color, 0.1);
     font-weight: 300;
     color: $white;
     width: 100%;
@@ -295,8 +300,8 @@
     -webkit-animation-fill-mode: both;
   }
 
-  .text-input::Placeholder {
-    color: rgba($white, .4);
+  .text-input::placeholder {
+    color: rgba($white, 0.4);
     line-height: 1.3em;
   }
 
@@ -328,7 +333,7 @@
       position: absolute;
       top: -2rem;
       opacity: 0;
-      transition: opacity .3s;
+      transition: opacity 0.3s;
     }
 
     .text-input {
@@ -338,8 +343,8 @@
       width: 100%;
     }
 
-    .text-input::Placeholder {
-      color: rgba($white, .4);
+    .text-input::placeholder {
+      color: rgba($white, 0.4);
     }
 
     .text-input:valid + .form-label {
diff --git a/aether-ui/aether/ui/assets/css/_section-entity-types.scss b/aether-ui/aether/ui/assets/css/_section-entity-types.scss
index d9c35e1e1..d961b0902 100644
--- a/aether-ui/aether/ui/assets/css/_section-entity-types.scss
+++ b/aether-ui/aether/ui/assets/css/_section-entity-types.scss
@@ -38,8 +38,8 @@
     flex: 1;
     display: none;
     padding: $unit $unit-xl;
-    border-left: 1px dashed rgba($light-grey, .4);
-    box-shadow: -1px 0 0 rgba($black, .3);
+    border-left: 1px dashed rgba($light-grey, 0.4);
+    box-shadow: -1px 0 0 rgba($black, 0.3);
   }
 
   &::after {
@@ -58,7 +58,7 @@
     display: flex;
     flex-direction: column;
     width: 100%;
-    height: calc(100vh - 6rem - #{$navbar-height-xs} - #{$unit}*2);
+    height: calc(100vh - 6rem - #{$navbar-height-xs} - #{$unit} * 2);
     min-height: 25em;
     position: relative;
     justify-content: space-between;
@@ -66,8 +66,10 @@
   }
 
   .textarea-header {
-    box-shadow: 1px 1px 1px rgba($black, .2) inset, 1px 0 0 rgba($white, .1);
     background: $dark-blue;
+    box-shadow:
+      1px 1px 1px rgba($black, 0.2) inset,
+      1px 0 0 rgba($white, 0.1);
   }
 
   textarea {
@@ -86,6 +88,7 @@
 
 .entity-type {
   @include card;
+
   margin-bottom: $unit;
   font-size: $font-size-xs;
   padding: $indent 0;
@@ -93,21 +96,21 @@
   .title {
     padding-left: $indent;
     padding-right: $indent;
-    margin-bottom: .6rem;
+    margin-bottom: 0.6rem;
     font-size: $font-size-m;
     font-weight: 600;
     overflow-wrap: break-word;
   }
 
   .field {
-    padding: .1rem 1rem;
+    padding: 0.1rem 1rem;
     border-bottom: 1px solid $white;
   }
 
   i {
-    margin-right: .4em;
+    margin-right: 0.4em;
     color: $text-color;
-    font-size: .8em;
+    font-size: 0.8em;
   }
 
   .name {
@@ -116,7 +119,7 @@
   }
 
   .type {
-    margin-left: .5rem;
+    margin-left: 0.5rem;
     font-style: italic;
     font-size: $font-size-xs;
     color: $grey;
@@ -125,7 +128,7 @@
 
 .pipeline--entities .pipeline-section--entities .section-right {
   display: block;
-  animation: show .5s;
+  animation: show 0.5s;
 }
 
 @media screen and (min-width: 1200px) {
diff --git a/aether-ui/aether/ui/assets/css/_section-input.scss b/aether-ui/aether/ui/assets/css/_section-input.scss
index 4970b4c2f..c6564d141 100644
--- a/aether-ui/aether/ui/assets/css/_section-input.scss
+++ b/aether-ui/aether/ui/assets/css/_section-input.scss
@@ -76,13 +76,13 @@
 
   i {
     color: $text-color;
-    font-size: .8em;
+    font-size: 0.8em;
     position: absolute;
-    top: .4rem;
+    top: 0.4rem;
   }
 
   i + .name {
-    padding-left: .8rem;
+    padding-left: 0.8rem;
   }
 
   .name {
@@ -90,7 +90,7 @@
   }
 
   .type {
-    margin-left: .5rem;
+    margin-left: 0.5rem;
     font-style: italic;
     font-size: $font-size-xs;
     color: $grey;
@@ -109,7 +109,7 @@
     padding-bottom: 4px;
     position: relative;
 
-    &>.name {
+    & > .name {
       font-weight: 400;
       position: relative;
 
@@ -122,8 +122,8 @@
         border-right: 4px solid transparent;
         border-top: 5px solid $grey;
         position: absolute;
-        left: -.8rem;
-        top: .5rem;
+        left: -0.8rem;
+        top: 0.5rem;
       }
     }
   }
@@ -136,8 +136,8 @@
   .group .group {
     .field,
     .group-title {
-      padding-top: .1rem;
-      padding-bottom: .1rem;
+      padding-top: 0.1rem;
+      padding-bottom: 0.1rem;
     }
 
     div[class^='input-schema-mapped'] {
diff --git a/aether-ui/aether/ui/assets/css/_section-mapping.scss b/aether-ui/aether/ui/assets/css/_section-mapping.scss
index d788c5340..33e2e33c9 100644
--- a/aether-ui/aether/ui/assets/css/_section-mapping.scss
+++ b/aether-ui/aether/ui/assets/css/_section-mapping.scss
@@ -30,14 +30,14 @@
 
   .tabs {
     .tab.selected {
-      box-shadow: 1px 1px 1px rgba($black, .2) inset;
+      box-shadow: 1px 1px 1px rgba($black, 0.2) inset;
       background-color: darken($dark-blue, 3);
       color: $white;
     }
   }
 
   .rules {
-    box-shadow: 1px 1px 1px rgba($black, .2) inset;
+    box-shadow: 1px 1px 1px rgba($black, 0.2) inset;
     background-color: darken($dark-blue, 3);
     padding: $unit $unit-xl;
     flex: 1 1 auto;
@@ -48,11 +48,11 @@
   .rule {
     display: flex;
     flex-wrap: wrap;
-    margin: .6rem 0;
+    margin: 0.6rem 0;
     justify-content: space-between;
 
     input {
-      padding: .4rem .8rem;
+      padding: 0.4rem 0.8rem;
       height: 32px;
       background: none;
       box-shadow: none;
@@ -60,13 +60,13 @@
       position: relative;
       z-index: 1;
 
-      &::Placeholder {
-        color: rgba($white, .4);
+      &::placeholder {
+        color: rgba($white, 0.4);
       }
     }
 
     .btn {
-      margin: .2rem 0;
+      margin: 0.2rem 0;
       min-width: 120px;
     }
   }
@@ -81,8 +81,8 @@
     &::before {
       border-radius: 50%;
       background-color: $red;
-      width: .6em;
-      height: .6em;
+      width: 0.6em;
+      height: 0.6em;
       position: absolute;
       left: -1.2em;
       top: 1em;
@@ -93,9 +93,11 @@
 
   .rule-input {
     flex: 1 1 auto;
-    margin: .2rem .6rem .2rem 0;
+    margin: 0.2rem 0.6rem 0.2rem 0;
     background-color: darken($dark-blue, 5);
-    box-shadow: 1px 1px 1px rgba($black, .2) inset, 1px 1px 0 rgba($white, .1);
+    box-shadow:
+      1px 1px 1px rgba($black, 0.2) inset,
+      1px 1px 0 rgba($white, 0.1);
     height: 32px;
     position: relative;
 
@@ -117,8 +119,8 @@
     &::before {
       right: -11px;
       background-color: darken($dark-blue, 5);
-      border-right: 1px solid rgba($white, .2);
-      border-top: 1px solid rgba($white, .1);
+      border-right: 1px solid rgba($white, 0.2);
+      border-top: 1px solid rgba($white, 0.1);
     }
   }
 
@@ -132,8 +134,8 @@
     &::before {
       left: -11px;
       background-color: darken($dark-blue, 3);
-      border-right: 1px solid rgba($black, .4);
-      border-top: 1px solid rgba($black, .3);
+      border-right: 1px solid rgba($black, 0.4);
+      border-top: 1px solid rgba($black, 0.3);
     }
   }
 
@@ -156,8 +158,10 @@
   }
 
   .textarea-header {
-    box-shadow: 1px 1px 1px rgba($black, .2) inset, 1px 0 0 rgba($white, .1);
     background: darken($dark-blue, 3);
+    box-shadow:
+      1px 1px 1px rgba($black, 0.2) inset,
+      1px 0 0 rgba($white, 0.1);
   }
 
   textarea {
diff --git a/aether-ui/aether/ui/assets/css/_section-output.scss b/aether-ui/aether/ui/assets/css/_section-output.scss
index 58a834b87..5e15bc3c5 100644
--- a/aether-ui/aether/ui/assets/css/_section-output.scss
+++ b/aether-ui/aether/ui/assets/css/_section-output.scss
@@ -21,7 +21,9 @@
 .pipeline-output {
   background: $text-color;
   color: $white;
-  box-shadow: -1px 0 0 rgba($black, .2), 1px 0 1px rgba($white, .3) inset;
+  box-shadow:
+    -1px 0 0 rgba($black, 0.2),
+    1px 0 1px rgba($white, 0.3) inset;
 
   .section-body {
     min-width: $output-width;
diff --git a/aether-ui/aether/ui/assets/css/base/_buttons.scss b/aether-ui/aether/ui/assets/css/base/_buttons.scss
index affd8f879..a73896f57 100644
--- a/aether-ui/aether/ui/assets/css/base/_buttons.scss
+++ b/aether-ui/aether/ui/assets/css/base/_buttons.scss
@@ -20,12 +20,13 @@
 
 .btn {
   @include label;
+
   border-radius: $border-radius;
-  background: rgba($grey, .1);
+  background: rgba($grey, 0.1);
   border: 0;
   border-color: transparent;
-  padding: .4em 2em;
-  transition: background .3s;
+  padding: 0.4em 2em;
+  transition: background 0.3s;
   color: $action-color;
 
   &:active,
@@ -38,9 +39,11 @@
   }
 
   &:disabled {
-    background: rgba($grey, .1);
+    background: rgba($grey, 0.1);
     color: $grey;
-    box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($text-color, .1) inset;
+    box-shadow:
+      1px 1px 0 rgba($white, 0.1) inset,
+      -1px -1px 0 rgba($text-color, 0.1) inset;
     cursor: default;
     pointer-events: none;
   }
@@ -50,8 +53,9 @@
 
 .btn-c {
   @include shadow-emboss-color;
+
   color: $white;
-  background: rgba($white, .15);
+  background: rgba($white, 0.15);
 
   &:active,
   &:focus,
@@ -60,8 +64,10 @@
   }
 
   &:disabled {
-    color: rgba($white, .5);
-    box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($black, .2) inset;
+    color: rgba($white, 0.5);
+    box-shadow:
+      1px 1px 0 rgba($white, 0.1) inset,
+      -1px -1px 0 rgba($black, 0.2) inset;
   }
 }
 
@@ -73,8 +79,9 @@
 
 .btn-d {
   @include shadow-emboss-dark;
+
   color: $action-color-b;
-  background: rgba($action-color-b, .15);
+  background: rgba($action-color-b, 0.15);
 
   &:active,
   &:focus,
@@ -83,7 +90,9 @@
   }
 
   &:disabled {
-    box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($black, .2) inset;
+    box-shadow:
+      1px 1px 0 rgba($white, 0.1) inset,
+      -1px -1px 0 rgba($black, 0.2) inset;
   }
 }
 
@@ -95,6 +104,7 @@
 
 .btn-w {
   @include shadow-emboss-white;
+
   color: $action-color;
 
   &:active,
@@ -117,6 +127,7 @@
   &:not(:disabled):not(.disabled):active:focus,
   &:hover {
     @include shadow-emboss-dark;
+
     background: $action-color-b;
   }
 }
@@ -138,7 +149,7 @@
   &:active,
   &:focus,
   &:hover {
-    background-color: rgba($dark-blue, .1);
+    background-color: rgba($dark-blue, 0.1);
     color: inherit;
   }
 }
@@ -149,11 +160,11 @@
 }
 
 .btn-sm {
-  padding: .2em .5em;
+  padding: 0.2em 0.5em;
 }
 
 .btn-square {
-  padding: .4em;
+  padding: 0.4em;
 
   i {
     display: block;
@@ -172,18 +183,20 @@
 
   i {
     @include shadow-flat-dark;
+
     text-align: center;
     display: block;
     width: 1.8rem;
     height: 1.8rem;
-    padding: .4em;
+    padding: 0.4em;
     border-radius: 50%;
     font-size: $font-size-s;
   }
 
   span {
     @include label;
-    margin-left: .5em;
+
+    margin-left: 0.5em;
   }
 
   &:hover {
diff --git a/aether-ui/aether/ui/assets/css/base/_color-codes.scss b/aether-ui/aether/ui/assets/css/base/_color-codes.scss
index 42eea474e..49a9af088 100644
--- a/aether-ui/aether/ui/assets/css/base/_color-codes.scss
+++ b/aether-ui/aether/ui/assets/css/base/_color-codes.scss
@@ -45,16 +45,16 @@ $colors-list: (
   }
 
   .entity-type-mapped-#{$i} {
-    background-color: rgba($current-color, .2);
+    background-color: rgba($current-color, 0.2);
     position: relative;
 
     &::after {
-      border-right: .6rem solid rgba($current-color, .2);
+      border-right: 0.6rem solid rgba($current-color, 0.2);
       content: '';
       display: block;
       position: absolute;
       top: 0;
-      left: -.6rem;
+      left: -0.6rem;
       width: 0;
       height: 0;
       border-top: $font-size-xs solid transparent;
@@ -69,13 +69,13 @@ $colors-list: (
       }
 
       &::after {
-        border-right: .6rem solid rgba($current-color, 1);
+        border-right: 0.6rem solid rgba($current-color, 1);
       }
     }
   }
 
   .input-schema-mapped-#{$i} {
-    background-color: rgba($current-color, .2);
+    background-color: rgba($current-color, 0.2);
     position: relative;
 
     &:hover {
@@ -90,12 +90,12 @@ $colors-list: (
       }
 
       &::after {
-        border-left: .6rem solid rgba($current-color, 1);
+        border-left: 0.6rem solid rgba($current-color, 1);
       }
     }
 
     &::before {
-      background-color: rgba($current-color, .2);
+      background-color: rgba($current-color, 0.2);
       content: '';
       display: block;
       position: absolute;
@@ -103,16 +103,16 @@ $colors-list: (
       left: -1rem;
       width: 1rem;
       height: 100%;
-      border-radius: .5rem 0 0 .5rem;
+      border-radius: 0.5rem 0 0 0.5rem;
     }
 
     &::after {
-      border-left: .6rem solid rgba($current-color, .2);
+      border-left: 0.6rem solid rgba($current-color, 0.2);
       content: '';
       display: block;
       position: absolute;
       top: 0;
-      right: -.6rem;
+      right: -0.6rem;
       width: 0;
       height: 0;
       border-top: $font-size-xs solid transparent;
diff --git a/aether-ui/aether/ui/assets/css/base/_fonts.scss b/aether-ui/aether/ui/assets/css/base/_fonts.scss
index b3d8f3755..91ec9a224 100644
--- a/aether-ui/aether/ui/assets/css/base/_fonts.scss
+++ b/aether-ui/aether/ui/assets/css/base/_fonts.scss
@@ -18,15 +18,10 @@
  * under the License.
  */
 
-// sass-lint:disable no-url-domains no-url-protocols
-
 /* Google Fonts */
 @import url('https://fonts.googleapis.com/css?family=Open+Sans:300,400,400i,600');
 @import url('https://fonts.googleapis.com/css?family=Fira+Mono');
 
-
 /* Font Awesome 5  https://fontawesome.com/ */
 @import url('https://use.fontawesome.com/releases/v5.13.0/css/fontawesome.css');
 @import url('https://use.fontawesome.com/releases/v5.13.0/css/solid.css');
-
-// sass-lint:disable no-url-domains no-url-protocols
diff --git a/aether-ui/aether/ui/assets/css/base/_global.scss b/aether-ui/aether/ui/assets/css/base/_global.scss
index 564245c9e..8ba37643a 100644
--- a/aether-ui/aether/ui/assets/css/base/_global.scss
+++ b/aether-ui/aether/ui/assets/css/base/_global.scss
@@ -39,7 +39,7 @@ a {
 }
 
 p {
-  margin-bottom: .5rem;
+  margin-bottom: 0.5rem;
 }
 
 ul {
@@ -60,20 +60,20 @@ small {
 }
 
 .badge {
-  background: rgba($grey, .1);
+  background: rgba($grey, 0.1);
   border-radius: $font-size-standard;
-  padding: .3rem .6rem;
+  padding: 0.3rem 0.6rem;
   font-weight: 300;
   font-size: $font-size-standard;
   text-align: center;
-  margin: 0 .4em;
+  margin: 0 0.4em;
   display: inline-block;
 }
 
 .badge-big {
   font-size: $font-size-l;
   border-radius: $font-size-l;
-  padding: .6rem .9rem;
+  padding: 0.6rem 0.9rem;
 }
 
 .badge-circle {
@@ -89,7 +89,8 @@ small {
 
 .badge-c {
   @include shadow-cutout-white;
-  background: rgba($text-color, .2);
+
+  background: rgba($text-color, 0.2);
 }
 
 .tag {
@@ -99,13 +100,13 @@ small {
   color: $white;
   background: $grey;
   border-radius: 1rem;
-  padding: 0 .4em;
+  padding: 0 0.4em;
 }
 
 .form-label {
   display: block;
   font-weight: 300;
-  letter-spacing: .04em;
+  letter-spacing: 0.04em;
   font-size: $font-size-m;
   margin-bottom: 1.2rem;
 }
@@ -116,6 +117,7 @@ small {
 
 .title-large {
   @extend .form-label;
+
   font-size: $font-size-l;
 }
 
@@ -123,37 +125,41 @@ input[type='password'],
 input[type='text'],
 textarea {
   @include shadow-cutout-white;
+
   border: 0;
   border-radius: $border-radius;
   background: $background-color;
   font-family: $body-font-family;
   color: $dark-blue;
-  padding: .8em;
+  padding: 0.8em;
 
   &:disabled {
     @include stripy($background-color, $white);
+
     box-shadow: none;
     color: $grey;
   }
 
   &.input-d {
     @include shadow-cutout-dark;
-    background: rgba($text-color, .3);
+
+    background: rgba($text-color, 0.3);
     color: $white;
 
-    &::Placeholder {
-      color: rgba($white, .4);
+    &::placeholder {
+      color: rgba($white, 0.4);
     }
 
     &:disabled {
-      @include stripy(rgba($text-color, .3), transparent);
+      @include stripy(rgba($text-color, 0.3), transparent);
+
       box-shadow: none;
       color: $grey;
     }
   }
 
   &.error {
-    background: rgba($red, .1);
+    background: rgba($red, 0.1);
   }
 
   &.input-large {
@@ -181,10 +187,11 @@ select:-webkit-autofill {
 
 .textarea-header {
   @include shadow-cutout-white;
+
   background: $light-grey;
   border-radius: $border-radius $border-radius 0 0;
   width: 100%;
-  min-height: .6rem;
+  min-height: 0.6rem;
   margin-bottom: -$border-radius;
   position: relative;
   z-index: 0;
@@ -219,16 +226,17 @@ select:-webkit-autofill {
 
 .options {
   @include card;
-  box-shadow: 2px 2px 3px rgba($text-color, .5);
+
+  box-shadow: 2px 2px 3px rgba($text-color, 0.5);
   position: absolute;
   margin: -5px 0 0 -5px;
-  padding: .6em 0;
+  padding: 0.6em 0;
   z-index: 3;
   min-width: 190px;
   list-style: none;
 
   li {
-    padding: .3em 1em;
+    padding: 0.3em 1em;
     cursor: pointer;
 
     &:hover {
@@ -267,6 +275,7 @@ select:-webkit-autofill {
 
   .tab.selected {
     @include shadow-cutout-white;
+
     background: $light-grey;
     border-radius: $border-radius $border-radius 0 0;
     color: $text-color;
@@ -276,12 +285,13 @@ select:-webkit-autofill {
 
 .status {
   @include shadow-cutout-dark;
+
   border-radius: 50%;
-  width: .5em;
-  height: .5em;
+  width: 0.5em;
+  height: 0.5em;
   content: '';
   display: inline-block;
-  margin-left: .5em;
+  margin-left: 0.5em;
 }
 
 .red {
@@ -306,7 +316,7 @@ pre {
   max-width: 500px;
   overflow: auto;
   white-space: pre;
-  padding: .5rem;
+  padding: 0.5rem;
 }
 
 code {
@@ -327,32 +337,32 @@ code {
     cursor: pointer;
     margin: 0;
     padding-left: 2.4rem;
-  }
 
-  label::before {
-    content: '';
-    display: block;
-    position: absolute;
-    left: 0;
-    top: 3px;
-    width: 26px;
-    height: 16px;
-    background-color: rgba($black, .2);
-    box-shadow: 0 0 2px rgba($black, .2) inset;
-    border-radius: 12px;
-  }
+    &::before {
+      content: '';
+      display: block;
+      position: absolute;
+      left: 0;
+      top: 3px;
+      width: 26px;
+      height: 16px;
+      background-color: rgba($black, 0.2);
+      box-shadow: 0 0 2px rgba($black, 0.2) inset;
+      border-radius: 12px;
+    }
 
-  label::after {
-    content: '';
-    display: block;
-    position: absolute;
-    top: 4px;
-    left: 1px;
-    width: 14px;
-    height: 14px;
-    border-radius: 50%;
-    background: $white;
-    transition: all .2s;
+    &::after {
+      content: '';
+      display: block;
+      position: absolute;
+      top: 4px;
+      left: 1px;
+      width: 14px;
+      height: 14px;
+      border-radius: 50%;
+      background: $white;
+      transition: all 0.2s;
+    }
   }
 
   input {
@@ -376,7 +386,7 @@ code {
   margin: 0;
   display: inline-block;
   position: relative;
-  margin-bottom: .8rem;
+  margin-bottom: 0.8rem;
 
   label {
     align-items: center;
@@ -385,22 +395,22 @@ code {
     margin: 0;
     font-size: $font-size-s;
     line-height: 1.3em;
-  }
 
-  label::before {
-    content: '\f00c';
-    font-family: $icon-font;
-    font-weight: 900;
-    font-size: 10px;
-    line-height: 12px;
-    display: block;
-    text-align: center;
-    color: transparent;
-    width: 16px;
-    height: 16px;
-    margin-right: .6rem;
-    border-radius: 16px;
-    border: 2px solid $action-color-b;
+    &::before {
+      content: '\f00c';
+      font-family: $icon-font;
+      font-weight: 900;
+      font-size: 10px;
+      line-height: 12px;
+      display: block;
+      text-align: center;
+      color: transparent;
+      width: 16px;
+      height: 16px;
+      margin-right: 0.6rem;
+      border-radius: 16px;
+      border: 2px solid $action-color-b;
+    }
   }
 
   input {
@@ -408,13 +418,11 @@ code {
   }
 
   input:checked + label {
-
     &::before {
       background: $action-color;
       border-color: $action-color;
       color: $white;
     }
-
   }
 }
 
@@ -429,7 +437,6 @@ code {
       border-color: transparent;
       color: $text-color;
     }
-
   }
 }
 
@@ -439,11 +446,11 @@ code {
   &::before {
     content: '';
     display: block;
-    width: .8rem;
+    width: 0.8rem;
     height: 1rem;
     position: absolute;
     left: 7px;
-    top: -.4rem;
+    top: -0.4rem;
     border-left: 2px solid $action-color-b;
     border-bottom: 2px solid $action-color-b;
   }
diff --git a/aether-ui/aether/ui/assets/css/base/_keyframes.scss b/aether-ui/aether/ui/assets/css/base/_keyframes.scss
index 4b4f84787..ea1b9f566 100644
--- a/aether-ui/aether/ui/assets/css/base/_keyframes.scss
+++ b/aether-ui/aether/ui/assets/css/base/_keyframes.scss
@@ -18,6 +18,8 @@
  * under the License.
  */
 
+// stylelint-disable block-opening-brace-space-before
+
 @keyframes fade-in {
   0%   { opacity: 0; }
   100% { opacity: 1; }
@@ -58,6 +60,8 @@
 @-webkit-keyframes autofill-dark {
   to {
     color: $white;
-    background: rgba($text-color, .1);
+    background: rgba($text-color, 0.1);
   }
 }
+
+// stylelint-enable block-opening-brace-space-before
diff --git a/aether-ui/aether/ui/assets/css/base/_loading-spinner.scss b/aether-ui/aether/ui/assets/css/base/_loading-spinner.scss
index 4d40fa525..fce2547ac 100644
--- a/aether-ui/aether/ui/assets/css/base/_loading-spinner.scss
+++ b/aether-ui/aether/ui/assets/css/base/_loading-spinner.scss
@@ -21,7 +21,7 @@
 $duration: 1.8s;
 
 .loading-spinner {
-  background-color: rgba($text-color, .7);
+  background-color: rgba($text-color, 0.7);
   position: fixed;
   top: 0;
   left: 0;
@@ -30,14 +30,20 @@ $duration: 1.8s;
   z-index: 999;
 }
 
-// sass-lint:disable one-declaration-per-line
+/* stylelint-disable
+    block-opening-brace-space-before,
+    declaration-block-single-line-max-declarations
+*/
 @keyframes dot {
   0%   { opacity: 0; top: 20px; background: $purple; }
   10%  { opacity: 1; }
   30%  { opacity: 1; top: 5px; background: $action-color; }
   100% { opacity: 0; top: 5px; background: $action-color; }
 }
-// sass-lint:enable one-declaration-per-line
+/* stylelint-enable
+    block-opening-brace-space-before,
+    declaration-block-single-line-max-declarations
+*/
 
 .dot1,
 .dot2,
@@ -64,7 +70,6 @@ $duration: 1.8s;
     opacity: 0;
     animation: dot $duration infinite;
   }
-
 }
 
 .dot2 {
diff --git a/aether-ui/aether/ui/assets/css/base/_messages.scss b/aether-ui/aether/ui/assets/css/base/_messages.scss
index 878acd6fb..6e8ae7600 100644
--- a/aether-ui/aether/ui/assets/css/base/_messages.scss
+++ b/aether-ui/aether/ui/assets/css/base/_messages.scss
@@ -29,10 +29,10 @@
 
 .hint.error-message {
   border-radius: $border-radius * 4;
-  background: rgba($red, .1);
+  background: rgba($red, 0.1);
   color: $red;
 }
 
 form .error-message + textarea {
-  background: rgba($red, .1);
+  background: rgba($red, 0.1);
 }
diff --git a/aether-ui/aether/ui/assets/css/base/_mixins.scss b/aether-ui/aether/ui/assets/css/base/_mixins.scss
index fcb8a2174..fbcfbed85 100644
--- a/aether-ui/aether/ui/assets/css/base/_mixins.scss
+++ b/aether-ui/aether/ui/assets/css/base/_mixins.scss
@@ -21,7 +21,7 @@
 @mixin label {
   font-size: $font-size-s;
   font-weight: 400;
-  letter-spacing: .06em;
+  letter-spacing: 0.06em;
   text-transform: uppercase;
 }
 
@@ -40,42 +40,65 @@
 }
 
 @mixin stripy($bg, $stripe) {
-  background: repeating-linear-gradient(
-    -45deg,
-    $stripe,
-    $stripe 10%,
-    $bg 10%,
-    $bg 50%,
-    $stripe 50%) top left fixed;
+  background:
+    repeating-linear-gradient(
+      -45deg,
+      $stripe,
+      $stripe 10%,
+      $bg 10%,
+      $bg 50%,
+      $stripe 50%
+    ) top left fixed;
   background-size: 8px 8px;
 }
 
 @mixin shadow-flat-dark {
-  box-shadow: -1px -1px 0 rgba($black, .2), 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($black, .2) inset, 1px 1px 0 rgba($white, .1);
+  box-shadow:
+    -1px -1px 0 rgba($black, 0.2),
+    1px 1px 0 rgba($white, 0.1) inset,
+    -1px -1px 0 rgba($black, 0.2) inset,
+    1px 1px 0 rgba($white, 0.1);
 }
 
 @mixin shadow-emboss-dark {
-  box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($black, .2) inset, 0 0 4px rgba($black, .3);
+  box-shadow:
+    1px 1px 0 rgba($white, 0.1) inset,
+    -1px -1px 0 rgba($black, 0.2) inset,
+    0 0 4px rgba($black, 0.3);
 }
 
 @mixin shadow-cutout-dark {
-  box-shadow: 1px 1px 1px rgba($black, .2) inset, 1px 1px 0 rgba($white, .1);
+  box-shadow:
+    1px 1px 1px rgba($black, 0.2) inset,
+    1px 1px 0 rgba($white, 0.1);
 }
 
 @mixin shadow-flat-white {
-  box-shadow: -1px -1px 0 rgba($text-color, .1), 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($text-color, .1) inset, 1px 1px 0 rgba($white, .1);
+  box-shadow:
+    -1px -1px 0 rgba($text-color, 0.1),
+    1px 1px 0 rgba($white, 0.1) inset,
+    -1px -1px 0 rgba($text-color, 0.1) inset,
+    1px 1px 0 rgba($white, 0.1);
 }
 
 @mixin shadow-emboss-white {
-  box-shadow: 1px 1px 0 rgba($white, .2) inset, -1px -1px 0 rgba($text-color, .2) inset, 0 0 2px rgba($text-color, .1);
+  box-shadow:
+    1px 1px 0 rgba($white, 0.2) inset,
+    -1px -1px 0 rgba($text-color, 0.2) inset,
+    0 0 2px rgba($text-color, 0.1);
 }
 
 @mixin shadow-cutout-white {
-  box-shadow: 1px 1px 1px rgba($text-color, .1) inset, 1px 1px 0 rgba($white, .1);
+  box-shadow:
+    1px 1px 1px rgba($text-color, 0.1) inset,
+    1px 1px 0 rgba($white, 0.1);
 }
 
 @mixin shadow-emboss-color {
-  box-shadow: 1px 1px 0 rgba($white, .1) inset, -1px -1px 0 rgba($text-color, .3) inset, 0 0 4px rgba($text-color, .1);
+  box-shadow:
+    1px 1px 0 rgba($white, 0.1) inset,
+    -1px -1px 0 rgba($text-color, 0.3) inset,
+    0 0 4px rgba($text-color, 0.1);
 }
 
 // STATES
@@ -83,6 +106,7 @@
 @mixin selected {
   .badge {
     @include bg-gradient;
+
     opacity: 1;
 
     .fa-caret-right {
diff --git a/aether-ui/aether/ui/assets/css/base/_modal.scss b/aether-ui/aether/ui/assets/css/base/_modal.scss
index 055c3dae0..8d46fabc2 100644
--- a/aether-ui/aether/ui/assets/css/base/_modal.scss
+++ b/aether-ui/aether/ui/assets/css/base/_modal.scss
@@ -19,7 +19,7 @@
  */
 
 .modal {
-  background-color: rgba($text-color, .7);
+  background-color: rgba($text-color, 0.7);
   color: $text-color;
   overflow-y: auto;
 
@@ -34,12 +34,13 @@
 
   .modal-title {
     font-weight: 300;
-    letter-spacing: .04em;
+    letter-spacing: 0.04em;
     font-size: $font-size-l;
   }
 
   .modal-dialog {
     @include card;
+
     margin: 10vh auto;
     padding: 0;
   }
@@ -53,7 +54,7 @@
     }
 
     li {
-      padding: .2rem 0;
+      padding: 0.2rem 0;
       list-style: none;
       font-size: $font-size-s;
 
@@ -63,7 +64,7 @@
         &::before {
           content: '\f06a'; // fa-exclamation-circle
           font: 1rem $icon-font;
-          margin-right: .3rem;
+          margin-right: 0.3rem;
         }
       }
 
@@ -73,7 +74,7 @@
         &::before {
           content: '\f071'; // fa-exclamation-triangle
           font: 1rem $icon-font;
-          margin-right: .3rem;
+          margin-right: 0.3rem;
         }
       }
 
@@ -83,7 +84,7 @@
         &::before {
           content: '\f058'; // fa-check-circle
           font: 1rem $icon-font;
-          margin-right: .3rem;
+          margin-right: 0.3rem;
         }
       }
     }
diff --git a/aether-ui/aether/ui/assets/css/base/_nav-bar.scss b/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
index b3eef8ceb..0c8c3e4c0 100644
--- a/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
+++ b/aether-ui/aether/ui/assets/css/base/_nav-bar.scss
@@ -30,8 +30,8 @@
 }
 
 .top-nav-breadcrumb {
-  animation: fade-in .6s;
-  letter-spacing: .06em;
+  animation: fade-in 0.6s;
+  letter-spacing: 0.06em;
   font-size: $font-size-s;
   display: flex;
   align-items: baseline;
@@ -39,8 +39,8 @@
 
   .breadcrumb-links a {
     display: inline-block;
-    padding-right: .2em;
-    transition: opacity .4s .4s, max-width .4s .4s;
+    padding-right: 0.2em;
+    transition: opacity 0.4s 0.4s, max-width 0.4s 0.4s;
   }
 
   a {
@@ -53,13 +53,14 @@
   }
 
   .tag {
-    margin-left: .4rem;
+    margin-left: 0.4rem;
   }
 }
 
 .top-nav-user {
   @include label;
-  transition: opacity .4s;
+
+  transition: opacity 0.4s;
   margin-left: auto;
 
   .user-name {
@@ -67,7 +68,7 @@
   }
 
   .logout {
-    margin-left: .7rem;
+    margin-left: 0.7rem;
     font-size: $font-size-standard;
   }
 }
@@ -77,7 +78,7 @@
   position: relative;
   width: 2.4rem;
   height: 1.2rem;
-  transition: opacity .4s .4s;
+  transition: opacity 0.4s 0.4s;
 
   span {
     display: inline-block;
@@ -94,7 +95,7 @@
 
 .show-pipeline {
   .top-nav {
-    padding: .3rem 1rem;
+    padding: 0.3rem 1rem;
   }
 
   .top-nav:hover {
diff --git a/aether-ui/aether/ui/assets/css/base/_settings.scss b/aether-ui/aether/ui/assets/css/base/_settings.scss
index 8be18dba8..b356c873a 100644
--- a/aether-ui/aether/ui/assets/css/base/_settings.scss
+++ b/aether-ui/aether/ui/assets/css/base/_settings.scss
@@ -39,7 +39,7 @@ $mid-blue:           lighten($dark-blue, 5);
 $grey:               #aea8b4;
 $light-grey:         #ede9f2;
 
-$shadow-color:       rgba($text-color, .3);
+$shadow-color:       rgba($text-color, 0.3);
 
 $action-color:       #0093ff;
 $action-color-b:     #50aef3;
@@ -49,13 +49,13 @@ $hover-color:        $purple;
 // Fonts
 // ************************************
 
-$body-font-family:   'Open Sans', Arial, sans-serif;
-$code-font-family:   'Fira Mono', SFMono-Regular, Menlo, Monaco, monospace;
+$body-font-family:   'Open Sans', arial, sans-serif;
+$code-font-family:   'Fira Mono', sfmono-regular, menlo, monaco, monospace;
 $icon-font:          'Font Awesome 5 Free';
 
-$font-size-xxs:      .7rem;
-$font-size-xs:       .86rem;
-$font-size-s:        .94rem;
+$font-size-xxs:      0.7rem;
+$font-size-xs:       0.86rem;
+$font-size-s:        0.94rem;
 $font-size-standard: 1rem;
 $font-size-m:        1.2rem;
 $font-size-l:        1.6rem;
@@ -76,4 +76,4 @@ $navbar-height-xs:   2.3rem;
 
 $indent:             1.2rem;
 
-$transition-speed:   .6s;
+$transition-speed:   0.6s;
diff --git a/aether-ui/aether/ui/assets/css/base/_styleguide.scss b/aether-ui/aether/ui/assets/css/base/_styleguide.scss
index f73838d73..24214dea0 100644
--- a/aether-ui/aether/ui/assets/css/base/_styleguide.scss
+++ b/aether-ui/aether/ui/assets/css/base/_styleguide.scss
@@ -33,32 +33,30 @@ $font-sizes: (
   xxl:       $font-size-xxl
 );
 
-// sass-lint:disable no-color-keywords
 $colors: (
   white:            $white,
   black:            $black,
-
+  //
   red:              $red,
   orange:           $orange,
   green:            $green,
-
+  //
   text-color:       $text-color,
   background-color: $background-color,
-
+  //
   dark-blue:        $dark-blue,
   purple:           $purple,
   light-grey:       $light-grey,
   grey:             $grey,
-
+  //
   shadow-color:     $shadow-color,
   action-color:     $action-color,
   hover-color:      $hover-color,
   action-color-b:   $action-color-b
 );
-// sass-lint:enable no-color-keywords
 
 .styleguide {
-  background-color: $white !important; // sass-lint:disable-line no-important
+  background-color: $white !important;
 
   .container {
     border-bottom: 1px dotted #4a4a4a;
diff --git a/aether-ui/aether/ui/assets/css/index.scss b/aether-ui/aether/ui/assets/css/index.scss
index 7fa1a36ac..564aad231 100644
--- a/aether-ui/aether/ui/assets/css/index.scss
+++ b/aether-ui/aether/ui/assets/css/index.scss
@@ -24,7 +24,6 @@
 
 @import '~bootstrap/scss/bootstrap';
 
-
 // ************************************
 // custom styles
 // ************************************
@@ -32,22 +31,22 @@
 @import 'base/fonts';
 @import 'base/settings';
 @import 'base/color-codes';
-@import 'base/mixins';
 @import 'base/keyframes';
+@import 'base/mixins';
 @import 'base/global';
 @import 'base/buttons';
 @import 'base/nav-bar';
 @import 'base/loading-spinner';
 @import 'base/modal';
 @import 'base/messages';
-
+//
 @import 'pipelines';
 @import 'pipeline';
 @import 'pipeline-settings';
-
+//
 @import 'section-input';
 @import 'section-entity-types';
 @import 'section-mapping';
 @import 'section-output';
-
+//
 @import 'base/styleguide';
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index f7b6e2a91..256953753 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -11,7 +11,7 @@
   },
   "scripts": {
     "test-lint-js": "standard './apps/**/*.js*'",
-    "test-lint-scss": "sass-lint --verbose",
+    "test-lint-scss": "stylelint './css/**/*.{css,scss,sass}'",
     "test-lint": "npm run test-lint-scss && npm run test-lint-js",
     "test-js": "jest --expand",
     "test-js-verbose": "TERM=dumb && jest --expand --colors --maxWorkers=1",
@@ -21,47 +21,49 @@
   },
   "dependencies": {
     "avsc": "~5.4.0",
-    "bootstrap": "~4.4.0",
+    "bootstrap": "~4.5.0",
     "html5shiv": "~3.7.0",
     "jquery": "~3.5.0",
-    "moment": "~2.24.0",
+    "moment": "~2.26.0",
     "popper.js": "~1.16.0",
     "react": "~16.13.0",
     "react-clipboard.js": "~2.0.16",
     "react-dom": "~16.13.0",
-    "react-intl": "~4.5.0",
+    "react-intl": "~4.6.0",
     "react-outside-click-handler": "~1.3.0",
     "react-redux": "~7.2.0",
-    "react-router-dom": "~5.1.0",
+    "react-router-dom": "~5.2.0",
     "redux": "~4.0.0",
     "redux-thunk": "~2.3.0",
-    "uuid": "~7.0.0",
+    "uuid": "~8.1.0",
     "webpack-google-cloud-storage-plugin": "~0.9.0",
     "webpack-s3-plugin": "~1.0.3",
     "whatwg-fetch": "~3.0.0"
   },
   "devDependencies": {
-    "@babel/core": "~7.9.0",
-    "@babel/plugin-proposal-class-properties": "~7.8.0",
-    "@babel/preset-env": "~7.9.0",
-    "@babel/preset-react": "~7.9.0",
+    "@babel/core": "~7.10.0",
+    "@babel/plugin-proposal-class-properties": "~7.10.0",
+    "@babel/preset-env": "~7.10.0",
+    "@babel/preset-react": "~7.10.0",
     "@hot-loader/react-dom": "~16.13.0",
     "babel-loader": "~8.1.0",
     "css-loader": "~3.5.0",
+    "eslint": "~7.1.0",
     "enzyme": "~3.11.0",
     "enzyme-adapter-react-16": "~1.15.0",
     "express": "~4.17.0",
-    "jest": "~25.4.0",
+    "jest": "~26.0.0",
     "mini-css-extract-plugin": "~0.9.0",
     "nock": "~12.0.0",
     "node-fetch": "~2.6.0",
     "node-sass": "~4.14.0",
     "react-hot-loader": "~4.12.0",
     "redux-devtools-extension": "~2.13.0",
-    "sass-lint": "~1.13.1",
     "sass-loader": "~8.0.0",
     "standard": "~14.3.0",
-    "style-loader": "~1.1.0",
+    "style-loader": "~1.2.0",
+    "stylelint": "~13.5.0",
+    "stylelint-config-standard": "~20.0.0",
     "webpack": "~4.43.0",
     "webpack-bundle-tracker": "~0.4.3",
     "webpack-cli": "~3.3.0",
@@ -78,10 +80,16 @@
       "@babel/plugin-proposal-class-properties"
     ]
   },
-  "sasslintConfig": "./conf/sass-lint.yml",
   "standard": {
     "verbose": true
   },
+  "stylelint": {
+    "extends": "stylelint-config-standard",
+    "rules": {
+      "at-rule-no-unknown": null,
+      "no-descending-specificity": null
+    }
+  },
   "jest": {
     "collectCoverage": true,
     "coverageDirectory": "<rootDir>/tests/.coverage",
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index 03ab2965c..b2bb59e9b 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -14,8 +14,8 @@
 
 aether.sdk==1.2.22
 autopep8==1.5.2
-boto3==1.12.48
-botocore==1.15.48
+boto3==1.13.19
+botocore==1.16.19
 cachetools==4.1.0
 certifi==2020.4.5.1
 cffi==1.14.0
@@ -24,14 +24,14 @@ configparser==5.0.0
 coverage==5.1
 cryptography==2.9.2
 Django==2.2.12
-django-cacheops==4.2
+django-cacheops==5.0
 django-cleanup==4.0.0
-django-cors-headers==3.2.1
+django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
 django-prometheus==2.0.0
-django-redis==4.11.0
+django-redis==4.12.1
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
@@ -39,43 +39,44 @@ django-webpack-loader==0.7.0
 djangorestframework==3.11.0
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
-entrypoints==0.3
-flake8==3.7.9
-flake8-quotes==3.0.0
+flake8==3.8.2
+flake8-quotes==3.2.0
 funcy==1.14
 google-api-core==1.17.0
-google-auth==1.14.1
+google-auth==1.16.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.0
+google-cloud-storage==1.28.1
 google-resumable-media==0.5.0
 googleapis-common-protos==1.51.0
 gprof2dot==2019.11.30
 idna==2.9
+importlib-metadata==1.6.0
 Jinja2==2.11.2
-jmespath==0.9.5
+jmespath==0.10.0
 MarkupSafe==1.1.1
 mccabe==0.6.1
 minio==5.0.10
-prometheus-client==0.7.1
-protobuf==3.11.3
+prometheus-client==0.8.0
+protobuf==3.12.2
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.5.0
+pycodestyle==2.6.0
 pycparser==2.20
-pyflakes==2.1.1
+pyflakes==2.2.0
 Pygments==2.6.1
 pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
-redis==3.4.1
+redis==3.5.2
 requests==2.23.0
 rsa==4.0
 s3transfer==0.3.3
-sentry-sdk==0.14.3
-six==1.14.0
+sentry-sdk==0.14.4
+six==1.15.0
 sqlparse==0.3.1
 tblib==1.6.0
 urllib3==1.25.9
 uWSGI==2.0.18
+zipp==3.1.0

From 8c762da7663bc53b6368ba003a6a4003f8cc057b Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 3 Jun 2020 09:14:55 +0200
Subject: [PATCH 07/29] test(kernel): use django-dynamic-fixture library (#837)

---
 .../aether/kernel/api/tests/test_filters.py   | 19 ++++---
 .../kernel/api/tests/utils/generators.py      | 53 ++++++++++---------
 .../conf/pip/primary-requirements.txt         |  2 +-
 aether-kernel/conf/pip/requirements.txt       |  2 +-
 4 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/tests/test_filters.py b/aether-kernel/aether/kernel/api/tests/test_filters.py
index 2159ae251..8b46d335a 100644
--- a/aether-kernel/aether/kernel/api/tests/test_filters.py
+++ b/aether-kernel/aether/kernel/api/tests/test_filters.py
@@ -18,16 +18,17 @@
 
 import json
 import random
-import string
 
-from autofixture import generators
 from django.contrib.auth import get_user_model
 from django.test import TestCase, override_settings
 from django.urls import reverse
 
 from aether.kernel.api import models
 from aether.kernel.api.filters import EntityFilter, SubmissionFilter
-from aether.kernel.api.tests.utils.generators import generate_project
+from aether.kernel.api.tests.utils.generators import (
+    generate_project,
+    generate_random_string,
+)
 
 
 @override_settings(MULTITENANCY=False)
@@ -261,9 +262,7 @@ def test_entity_filter__by_family(self):
         # Generate projects.
         for _ in range(random.randint(5, 10)):
             generate_project(schema_field_values={
-                # The filter test fails if the generated string ends with a space.
-                # The serializer class removes any trailing whitespace sent to the field.
-                'family': generators.StringGenerator(min_length=10, max_length=30, chars=string.ascii_letters),
+                'family': generate_random_string(),
             })
         entities_count = models.Entity.objects.count()
         # Get a list of all schema families.
@@ -553,9 +552,9 @@ def test_submission_filter__by_payload(self):
             {'a': {'b': 'abcde'}, 'z': 3},
             {'a': {'b': {'c': [1, 2, 3]}}, 'z': 3}
         ]
-        gen_payload = generators.ChoicesGenerator(values=payloads)
-        generate_project(submission_field_values={'payload': gen_payload})
+        generate_project(submission_field_values={'payload': lambda x: random.choice(payloads)})
         submissions_count = models.Submission.objects.count()
+        self.assertTrue(submissions_count > 0)
 
         filtered_submissions_count = 0
         for kwargs, payload in zip(filters, payloads):
@@ -593,9 +592,9 @@ def test_submission_filter__by_payload__post(self):
             {'a': {'b': 'abcde'}, 'z': 3},
             {'a': {'b': {'c': [1, 2, 3]}}, 'z': 3}
         ]
-        gen_payload = generators.ChoicesGenerator(values=payloads)
-        generate_project(submission_field_values={'payload': gen_payload})
+        generate_project(submission_field_values={'payload': lambda x: random.choice(payloads)})
         submissions_count = models.Submission.objects.count()
+        self.assertTrue(submissions_count > 0)
 
         filtered_submissions_count = 0
         for kwargs, payload in zip(filters, payloads):
diff --git a/aether-kernel/aether/kernel/api/tests/utils/generators.py b/aether-kernel/aether/kernel/api/tests/utils/generators.py
index 4db90a504..c430dc3d9 100644
--- a/aether-kernel/aether/kernel/api/tests/utils/generators.py
+++ b/aether-kernel/aether/kernel/api/tests/utils/generators.py
@@ -18,7 +18,7 @@
 
 import random
 
-from autofixture import AutoFixture
+from ddf import G, M
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 
@@ -93,8 +93,7 @@ def generate_project(
     Generate an Aether Project.
 
     This function can be used in unit tests to generate instances of all kernel
-    models. It wraps https://github.com/gregmuellegger/django-autofixture and
-    provides an Aether-specific fixture generator.
+    models.
 
     If necessary, the default field values of a model can be overridden,
     using either static values:
@@ -103,59 +102,57 @@ def generate_project(
 
     or generators:
 
-    >>> from autofixture import generators
-    >>> names = generators.ChoicesGenerator(values=['a', 'b', 'c'])
+    >>> import random
+    >>> names = lambda x: random.choice(['a', 'b', 'c'])
     >>> generate_project(project_field_values={'name': names})
     '''
 
-    project = AutoFixture(
+    project = G(
         models.Project,
-        field_values=get_field_values(
+        **get_field_values(
             default=dict(),
             values=project_field_values,
         ),
-    ).create_one()
+    )
 
-    schema = AutoFixture(
-        model=models.Schema,
-        field_values=get_field_values(
+    schema = G(
+        models.Schema,
+        **get_field_values(
             default=dict(
                 definition=schema_definition(),
             ),
             values=schema_field_values,
         ),
-    ).create_one()
+    )
 
-    schemadecorator = AutoFixture(
+    schemadecorator = G(
         model=models.SchemaDecorator,
-        field_values=get_field_values(
+        **get_field_values(
             default=dict(
                 project=project,
                 schema=schema,
             ),
             values=schemadecorator_field_values,
         ),
-    ).create_one()
+    )
 
     for _ in range(random.randint(*MAPPINGS_COUNT_RANGE)):
-        mappingset = AutoFixture(
-            model=models.MappingSet,
-            field_values=get_field_values(
+        mappingset = G(
+            models.MappingSet,
+            **get_field_values(
                 default=dict(
                     project=project,
                     schema=mappingset_schema(),
                 ),
                 values=mappingset_field_values,
             ),
-        ).create_one()
+        )
 
         # create a random input based on the schema
         if not mappingset.input and mappingset.schema:
             mappingset.input = random_avro(mappingset.schema)
             mappingset.save()
 
-        # django-autofixture does not support Django 2 models with m2m relations.
-        # https://github.com/gregmuellegger/django-autofixture/pull/110
         models.Mapping.objects.create(
             **get_field_values(
                 default=dict(
@@ -168,9 +165,9 @@ def generate_project(
         )
 
         for _ in range(random.randint(*SUBMISSIONS_COUNT_RANGE)):
-            submission = AutoFixture(
-                model=models.Submission,
-                field_values=get_field_values(
+            submission = G(
+                models.Submission,
+                **get_field_values(
                     default=dict(
                         # use mappingset schema to generate random payloads
                         payload=random_avro(mappingset.schema),
@@ -179,7 +176,7 @@ def generate_project(
                     ),
                     values=submission_field_values,
                 ),
-            ).create_one()
+            )
 
             if include_attachments:
                 for index in range(random.randint(*ATTACHMENTS_COUNT_RANGE)):
@@ -190,3 +187,9 @@ def generate_project(
 
             # extract entities
             run_entity_extraction(submission)
+
+    return project, schema
+
+
+def generate_random_string():
+    return M('---------------------')
diff --git a/aether-kernel/conf/pip/primary-requirements.txt b/aether-kernel/conf/pip/primary-requirements.txt
index c734482d7..f2bfa3fec 100644
--- a/aether-kernel/conf/pip/primary-requirements.txt
+++ b/aether-kernel/conf/pip/primary-requirements.txt
@@ -34,4 +34,4 @@ lxml
 
 
 # Test libraries
-django-autofixture
+django-dynamic-fixture
diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 2c9f27524..addf5746b 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -29,11 +29,11 @@ coverage==5.1
 cryptography==2.9.2
 decorator==4.4.2
 Django==2.2.12
-django-autofixture==0.12.1
 django-cacheops==5.0
 django-cleanup==4.0.0
 django-cors-headers==3.3.0
 django-debug-toolbar==2.2
+django-dynamic-fixture==3.1.0
 django-filter==2.2.0
 django-minio-storage==0.3.7
 django-model-utils==4.0.0

From 21dabf9a74a446a5df892cbf4a4a251bdfc16f9d Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 3 Jun 2020 09:15:37 +0200
Subject: [PATCH 08/29] fix: rethink error messages level (#839)

---
 aether-kernel/aether/kernel/api/exporter.py   |  8 ++++----
 .../aether/odk/api/collect/auth_utils.py      | 14 ++++++-------
 .../aether/odk/api/collect/views.py           | 12 +++++------
 aether-producer/producer/__init__.py          |  2 +-
 aether-producer/producer/db.py                | 20 +++++++++----------
 aether-producer/producer/kernel.py            |  2 +-
 aether-producer/producer/kernel_api.py        | 12 +++++------
 aether-producer/producer/kernel_db.py         | 14 ++++++-------
 aether-producer/producer/topic.py             | 16 +++++++--------
 9 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/exporter.py b/aether-kernel/aether/kernel/api/exporter.py
index bff760c33..39533a5de 100644
--- a/aether-kernel/aether/kernel/api/exporter.py
+++ b/aether-kernel/aether/kernel/api/exporter.py
@@ -548,7 +548,7 @@ def __exec(counter):
                 counter,
             )
         except Exception as e:
-            logger.error(f'Got an error while generating records file: {str(e)}')
+            logger.warning(f'Got an error while generating records file: {str(e)}')
             task.set_error_records(str(e))
             raise  # required to force exitcode != 0
 
@@ -637,7 +637,7 @@ def __exec(counter):
         logger.info(f'File "{file_name}" ready!')
 
     except Exception as e:
-        logger.error(f'Got an error while generating records file: {str(e)}')
+        logger.debug(f'Got an error while generating records file: {str(e)}')
         task.set_error_records(str(e))
         raise  # required to force exitcode != 0
 
@@ -686,7 +686,7 @@ def _fetch_attachs(_start, _stop, counter, counter_attachs):
                     data_from = data_to
 
         except Exception as e:
-            logger.error(f'Got an error while generating attachments file: {str(e)}')
+            logger.debug(f'Got an error while generating attachments file: {str(e)}')
             task.set_error_attachments(str(e))
             raise  # required to force exitcode != 0
 
@@ -801,7 +801,7 @@ def _fetch_attachs(_start, _stop, counter, counter_attachs):
             logger.info(f'File "{zip_name}.{zip_ext}" ready!')
 
     except Exception as e:
-        logger.error(f'Got an error while generating attachments file: {str(e)}')
+        logger.debug(f'Got an error while generating attachments file: {str(e)}')
         task.set_error_attachments(str(e))
 
 
diff --git a/aether-odk-module/aether/odk/api/collect/auth_utils.py b/aether-odk-module/aether/odk/api/collect/auth_utils.py
index b31a271c1..57daf229f 100644
--- a/aether-odk-module/aether/odk/api/collect/auth_utils.py
+++ b/aether-odk-module/aether/odk/api/collect/auth_utils.py
@@ -130,7 +130,7 @@ def check_authorization_header(request):
     for field in _REQUIRED_FIELDS:
         if field not in challenge:
             msg = MSG_MISSING.format(field)
-            logger.error(msg)
+            logger.debug(msg)
             raise AuthenticationFailed(msg)
 
     # check field values
@@ -144,19 +144,19 @@ def check_authorization_header(request):
     for field in ('algorithm', 'nonce', 'qop', 'realm'):
         if challenge[field] != expected_values[field]:
             msg = MSG_WRONG.format(field)
-            logger.error(msg)
+            logger.debug(msg)
             raise AuthenticationFailed(msg)
 
     # check URI
     if unquote(urlparse(challenge['uri']).path) != request.path:
         msg = MSG_WRONG.format('uri')
-        logger.error(msg)
+        logger.debug(msg)
         raise AuthenticationFailed(msg)
 
     # SECURITY ENHANCEMENT: check if the nonce timepstamp is not older than ...
     if float(timestamp) <= lifespan.timestamp():
         msg = MSG_WRONG.format('nonce')
-        logger.error(msg)
+        logger.debug(msg)
         raise AuthenticationFailed(msg)
 
     # check nonce counter
@@ -171,7 +171,7 @@ def check_authorization_header(request):
 
     current_counter = int(challenge['nc'], 16)
     if last_counter is not None and last_counter >= current_counter:
-        logger.error(MSG_COUNTER)
+        logger.debug(MSG_COUNTER)
         raise AuthenticationFailed(MSG_COUNTER)
 
     else:
@@ -188,13 +188,13 @@ def check_authorization_header(request):
         user = get_user_model().objects.get(username=parse_username(request, username))
         partial = DigestPartial.objects.get(user=user, username=username)
     except ObjectDoesNotExist:
-        logger.error(MSG_USERNAME)
+        logger.debug(MSG_USERNAME)
         raise AuthenticationFailed(MSG_USERNAME)
 
     # check header response
     response_hash = _generate_response(request, partial.digest, challenge)
     if response_hash != challenge['response']:
-        logger.error(MSG_HEADER)
+        logger.debug(MSG_HEADER)
         raise AuthenticationFailed(MSG_HEADER)
 
     return user
diff --git a/aether-odk-module/aether/odk/api/collect/views.py b/aether-odk-module/aether/odk/api/collect/views.py
index 2c99f485f..a015a6680 100644
--- a/aether-odk-module/aether/odk/api/collect/views.py
+++ b/aether-odk-module/aether/odk/api/collect/views.py
@@ -456,7 +456,7 @@ def _rollback_submission(submission_id):
     except Exception as e:
         msg = MSG_SUBMISSION_FILE_ERR
         logger.warning(msg)
-        logger.error(str(e))
+        logger.debug(str(e))
         return _send_response(
             request=request,
             nature=NATURE_SUBMIT_ERROR,
@@ -490,7 +490,7 @@ def _rollback_submission(submission_id):
     xforms = _get_xforms(request).filter(form_id=form_id)
     if not xforms.exists():
         msg = MSG_SUBMISSION_XFORM_NOT_FOUND_ERR.format(form_id=form_id)
-        logger.error(msg)
+        logger.debug(msg)
         return _send_response(
             request=request,
             nature=NATURE_SUBMIT_ERROR,
@@ -501,7 +501,7 @@ def _rollback_submission(submission_id):
     xforms = xforms.filter(project__active=True, active=True)
     if not xforms.exists():
         msg = MSG_INSTANCE_INACTIVE.format(form_id=form_id)
-        logger.error(msg)
+        logger.debug(msg)
         return _send_response(
             request=request,
             nature=NATURE_SUBMIT_ERROR,
@@ -519,7 +519,7 @@ def _rollback_submission(submission_id):
 
     if not xform:
         msg = MSG_SUBMISSION_XFORM_UNAUTHORIZED_ERR.format(form_id=form_id)
-        logger.error(msg)
+        logger.debug(msg)
         return _send_response(
             request=request,
             nature=NATURE_SUBMIT_ERROR,
@@ -541,7 +541,7 @@ def _rollback_submission(submission_id):
     except KernelPropagationError as kpe:
         msg = MSG_SUBMISSION_KERNEL_ARTEFACTS_ERR.format(form_id=form_id)
         logger.warning(msg)
-        logger.error(str(kpe))
+        logger.debug(str(kpe))
         return _send_response(
             request=request,
             nature=NATURE_SUBMIT_ERROR,
@@ -652,7 +652,7 @@ def _rollback_submission(submission_id):
     except Exception as e:
         msg = MSG_SUBMISSION_SUBMIT_ERR.format(form_id=form_id)
         logger.warning(msg)
-        logger.error(str(e))
+        logger.debug(str(e))
 
         # delete submission and ignore response
         _rollback_submission(submission_id)
diff --git a/aether-producer/producer/__init__.py b/aether-producer/producer/__init__.py
index 88c18b216..ee39da68e 100644
--- a/aether-producer/producer/__init__.py
+++ b/aether-producer/producer/__init__.py
@@ -157,7 +157,7 @@ def check_schemas(self):
             try:
                 schemas = self.kernel_client.get_schemas()
             except Exception as err:
-                self.logger.error(f'No Kernel connection: {err}')
+                self.logger.warning(f'No Kernel connection: {err}')
                 gevent.sleep(1)
                 continue
 
diff --git a/aether-producer/producer/db.py b/aether-producer/producer/db.py
index 09230bce1..a1929c43b 100644
--- a/aether-producer/producer/db.py
+++ b/aether-producer/producer/db.py
@@ -120,8 +120,8 @@ def _dispatcher(self):
                 logger.debug(f'{self.name} pulled 1 for {name}: still {len(self.connection_pool)}')
                 sleep(0)  # allow other coroutines to work
                 while not self._test_connection(conn):
-                    logger.error('Pooled connection is dead, getting new resource.'
-                                 ' Replacing dead pool member.')
+                    logger.warning('Pooled connection is dead, getting new resource.'
+                                   ' Replacing dead pool member.')
                     conn = self._make_connection()
                     sleep(0)
                 logger.debug(f'Got job from {name} @priority {priority_level}')
@@ -157,7 +157,7 @@ def _shutdown_pool(self):
                 conn.close()
                 logger.debug(f'{self.name} shutdown connection #{c}')
             except Exception as err:
-                logger.error(f'{self.name} FAILED to shutdown connection #{c} | err: {err}')
+                logger.warning(f'{self.name} FAILED to shutdown connection #{c} | err: {err}')
             finally:
                 c += 1
 
@@ -235,14 +235,14 @@ def update(cls, name, offset_value):
             return offset_value
 
         except Exception as err:
-            logger.error(err)
+            logger.warning(err)
             raise err
 
         finally:
             try:
                 Offset.__pool__.release(call, conn)
             except UnboundLocalError:
-                logger.error(f'{call} could not release a connection it never received.')
+                logger.warning(f'{call} could not release a connection it never received.')
 
     @classmethod
     def get_offset(cls, name):
@@ -257,14 +257,14 @@ def get_offset(cls, name):
             return res[0][0] if res else None
 
         except Exception as err:
-            logger.error(err)
+            logger.warning(err)
             raise err
 
         finally:
             try:
                 Offset.__pool__.release(call, conn)
             except UnboundLocalError:
-                logger.error(f'{call} could not release a connection it never received.')
+                logger.warning(f'{call} could not release a connection it never received.')
 
 
 def init():
@@ -280,7 +280,7 @@ def _start_session(engine):
             sessionmaker(bind=engine)
             logger.info('Database initialized.')
         except SQLAlchemyError as err:
-            logger.error(f'Database could not be initialized | {err}')
+            logger.warning(f'Database could not be initialized | {err}')
             raise err
 
     def _create_db():
@@ -305,7 +305,7 @@ def _create_db():
         Offset.create_pool()
         return
     except SQLAlchemyError as sqe:
-        logger.error(f'Start session failed (1st attempt): {sqe}')
+        logger.warning(f'Start session failed (1st attempt): {sqe}')
         pass
 
     # it was not possible to start session because the database does not exit
@@ -315,6 +315,6 @@ def _create_db():
         _start_session(engine)
         Offset.create_pool()
     except SQLAlchemyError as sqe:
-        logger.error(f'Start session failed (2nd attempt): {sqe}')
+        logger.critical(f'Start session failed (2nd attempt): {sqe}')
         logger.exception(sqe)
         sys.exit(1)
diff --git a/aether-producer/producer/kernel.py b/aether-producer/producer/kernel.py
index 6984bc6c1..d9595d87c 100644
--- a/aether-producer/producer/kernel.py
+++ b/aether-producer/producer/kernel.py
@@ -50,7 +50,7 @@ def fn(row):
 
             elif lag_time < -30.0:
                 # Sometimes fractional negatives show up. More than 30 seconds is an issue though.
-                logger.critical(f'INVALID LAG INTERVAL: {lag_time}. Check time settings on server.')
+                logger.warning(f'INVALID LAG INTERVAL: {lag_time}. Check time settings on server.')
 
             _id = row.get('id')
             logger.debug(f'WINDOW EXCLUDE: ID: {_id}, LAG: {lag_time}')
diff --git a/aether-producer/producer/kernel_api.py b/aether-producer/producer/kernel_api.py
index 7a29ffd4d..2f909c6b2 100644
--- a/aether-producer/producer/kernel_api.py
+++ b/aether-producer/producer/kernel_api.py
@@ -78,7 +78,7 @@ def get_schemas(self):
 
         except Exception:
             self.last_check_error = 'Could not access kernel API to get topics'
-            logger.critical(self.last_check_error)
+            logger.warning(self.last_check_error)
             return []
 
     def check_updates(self, realm, schema_id, schema_name, modified):
@@ -91,7 +91,7 @@ def check_updates(self, realm, schema_id, schema_name, modified):
             response = self._fetch(url=url, realm=realm)
             return response['count'] > 1
         except Exception:
-            logger.critical('Could not access kernel API to look for updates')
+            logger.warning('Could not access kernel API to look for updates')
             return False
 
     def count_updates(self, realm, schema_id, schema_name, modified=''):
@@ -105,7 +105,7 @@ def count_updates(self, realm, schema_id, schema_name, modified=''):
             logger.debug(f'Reporting requested size for {schema_name} of {_count}')
             return {'count': _count}
         except Exception:
-            logger.critical('Could not access kernel API to look for updates')
+            logger.warning('Could not access kernel API to look for updates')
             return -1
 
     def get_updates(self, realm, schema_id, schema_name, modified):
@@ -127,7 +127,7 @@ def get_updates(self, realm, schema_id, schema_name, modified):
             ]
 
         except Exception:
-            logger.critical('Could not access kernel API to look for updates')
+            logger.warning('Could not access kernel API to look for updates')
             return []
 
     def _fetch(self, url, realm=None):
@@ -153,7 +153,7 @@ def _fetch(self, url, realm=None):
                 return response.json()
             except Exception as e:
                 if count >= _REQUEST_ERROR_RETRIES:
-                    logger.error(f'Error while fetching data from {url}')
-                    logger.error(e)
+                    logger.warning(f'Error while fetching data from {url}')
+                    logger.debug(e)
                     raise e
             sleep(count)  # sleep longer in each iteration
diff --git a/aether-producer/producer/kernel_db.py b/aether-producer/producer/kernel_db.py
index 2f6676532..51adba35f 100644
--- a/aether-producer/producer/kernel_db.py
+++ b/aether-producer/producer/kernel_db.py
@@ -98,7 +98,7 @@ def get_schemas(self):
                 yield {key: row[key] for key in row.keys()}
         else:
             self.last_check_error = 'Could not access kernel database to get topics'
-            logger.critical('Could not access kernel database to get topics')
+            logger.warning('Could not access kernel database to get topics')
             return []
 
     def check_updates(self, realm, schema_id, schema_name, modified):
@@ -111,7 +111,7 @@ def check_updates(self, realm, schema_id, schema_name, modified):
         if cursor:
             return sum([1 for i in cursor]) > 0
         else:
-            logger.critical('Could not access kernel database to look for updates')
+            logger.warning('Could not access kernel database to look for updates')
             return False
 
     def count_updates(self, realm, schema_id, schema_name, modified=''):
@@ -132,7 +132,7 @@ def count_updates(self, realm, schema_id, schema_name, modified=''):
             logger.debug(f'Reporting requested size for {schema_name} of {_count}')
             return {'count': _count}
         else:
-            logger.critical('Could not access kernel database to look for updates')
+            logger.warning('Could not access kernel database to look for updates')
             return -1
 
     def get_updates(self, realm, schema_id, schema_name, modified):
@@ -153,7 +153,7 @@ def get_updates(self, realm, schema_id, schema_name, modified):
                 if window_filter(row)
             ]
         else:
-            logger.critical('Could not access kernel database to look for updates')
+            logger.warning('Could not access kernel database to look for updates')
             return []
 
     def _exec_sql(self, name, priority, query):
@@ -166,12 +166,12 @@ def _exec_sql(self, name, priority, query):
             return cursor
 
         except psycopg2.OperationalError as pgerr:
-            logger.critical(f'Error while accessing database: {pgerr}')
-            logger.exception(pgerr)
+            logger.warning(f'Error while accessing database: {pgerr}')
+            logger.debug(pgerr)
             return None
 
         finally:
             try:
                 self.pool.release(name, conn)
             except UnboundLocalError:
-                logger.error(f'{name} could not release a connection it never received.')
+                logger.warning(f'{name} could not release a connection it never received.')
diff --git a/aether-producer/producer/topic.py b/aether-producer/producer/topic.py
index a553941b3..3c726c2f7 100644
--- a/aether-producer/producer/topic.py
+++ b/aether-producer/producer/topic.py
@@ -131,7 +131,7 @@ def create_topic(self):
                 logger.info(f'Created topic {self.name}')
                 return True
             else:
-                logger.critical(f'Topic {self.name} could not be created: {e}')
+                logger.warning(f'Topic {self.name} could not be created: {e}')
                 return False
 
     def get_producer(self):
@@ -180,7 +180,7 @@ def handle_rebuild(self):
         self.producer = None
 
         if not self.delete_this_topic():
-            logger.critical(f'{tag} FAILED. Topic will not resume.')
+            logger.warning(f'{tag} FAILED. Topic will not resume.')
             self.operating_status = TopicStatus.LOCKED
             return
 
@@ -237,7 +237,7 @@ def get_status(self):
     # Callback function registered with Kafka Producer to acknowledge receipt
     def kafka_callback(self, err=None, msg=None, _=None, **kwargs):
         if err:
-            logger.error(f'ERROR [{err}, {msg}, {kwargs}]')
+            logger.warning(f'ERROR [{err}, {msg}, {kwargs}]')
 
         with io.BytesIO() as obj:
             obj.write(msg.value())
@@ -296,7 +296,7 @@ def update_kafka(self):
 
                 end_offset = new_rows[-1].get('modified')
             except Exception as pge:
-                logger.error(f'Could not get new records from kernel: {pge}')
+                logger.warning(f'Could not get new records from kernel: {pge}')
                 self.context.safe_sleep(self.sleep_time)
                 continue
 
@@ -317,7 +317,7 @@ def update_kafka(self):
                             writer.append(msg)
                         else:
                             # Message doesn't have the proper format for the current schema.
-                            logger.critical(
+                            logger.warning(
                                 f'SCHEMA_MISMATCH: NOT SAVED! TOPIC: {self.name}, ID: {_id}')
 
                     writer.flush()
@@ -333,8 +333,8 @@ def update_kafka(self):
                 self.wait_for_kafka(end_offset, failure_wait_time=self.kafka_failure_wait_time)
 
             except Exception as ke:
-                logger.error(f'error in Kafka save: {ke}')
-                logger.error(traceback.format_exc())
+                logger.warning(f'error in Kafka save: {ke}')
+                logger.warning(traceback.format_exc())
                 self.context.safe_sleep(self.sleep_time)
 
     def wait_for_kafka(self, end_offset, timeout=10, iters_per_sec=10, failure_wait_time=10):
@@ -413,7 +413,7 @@ def handle_kafka_errors(self, change_set_size, all_failed=False, failure_wait_ti
         if not all_failed:
             # Collect Error types for reporting
             for _id, err in self.failed_changes.items():
-                logger.critical(f'PRODUCER_FAILURE: T: {self.name} ID {_id}, ERR_MSG {err.name()}')
+                logger.warning(f'PRODUCER_FAILURE: T: {self.name} ID {_id}, ERR_MSG {err.name()}')
 
             dropped_messages = change_set_size - len(self.successful_changes)
             errors['NO_REPLY'] = dropped_messages - len(self.failed_changes)

From 1fa63dba47752549d15e904d627c7f92c9b2fe30 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 8 Jun 2020 09:45:29 +0200
Subject: [PATCH 09/29] chore: upgrade dependencies (#843)

* chore: upgrade dependencies

* chore: docker-compose 2.4 version

* fix: error handling within try/except

* fix: missing dc file

* chore: more upgrades

* chore: clean django 3 deps
---
 aether-client-library/docker-compose.yml    |  2 +-
 aether-kernel/aether/kernel/api/views.py    | 10 +++----
 aether-kernel/conf/pip/requirements.txt     | 31 +++++++++++----------
 aether-odk-module/conf/pip/requirements.txt | 27 +++++++++---------
 aether-producer/conf/pip/requirements.txt   | 12 ++++----
 aether-ui/aether/ui/assets/package.json     |  4 +--
 aether-ui/conf/pip/requirements.txt         | 27 +++++++++---------
 docker-compose-base.yml                     |  2 +-
 docker-compose-connect.yml                  |  2 +-
 docker-compose-test.yml                     |  2 +-
 docker-compose.yml                          |  2 +-
 11 files changed, 62 insertions(+), 59 deletions(-)

diff --git a/aether-client-library/docker-compose.yml b/aether-client-library/docker-compose.yml
index f14f515d2..89f735a0f 100644
--- a/aether-client-library/docker-compose.yml
+++ b/aether-client-library/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.1"
+version: "2.4"
 
 services:
 
diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py
index d17f8f56a..d82b67c86 100644
--- a/aether-kernel/aether/kernel/api/views.py
+++ b/aether-kernel/aether/kernel/api/views.py
@@ -559,12 +559,12 @@ def unique_usage(self, request, *args, **kwargs):
             # of the supplied mappings only
         }
         '''
-        mappings = filter_by_realm(
-            self.request,
-            models.Mapping.objects.filter(pk__in=request.data or []),
-            'mappingset__project'
-        ).values_list('id', flat=True)
         try:
+            mappings = filter_by_realm(
+                self.request,
+                models.Mapping.objects.filter(pk__in=request.data or []),
+                'mappingset__project'
+            ).values_list('id', flat=True)
             return Response(utils.get_unique_schemas_used(mappings))
         except Exception as e:
             return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index addf5746b..2c7a455d6 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -13,13 +13,13 @@
 ################################################################################
 
 aether.python==1.0.17
-aether.sdk==1.2.22
+aether.sdk==1.3.0
 attrs==19.3.0
-autopep8==1.5.2
-boto3==1.13.19
-botocore==1.16.19
+autopep8==1.5.3
+boto3==1.13.24
+botocore==1.16.24
 cachetools==4.1.0
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
@@ -28,13 +28,13 @@ coreschema==0.0.4
 coverage==5.1
 cryptography==2.9.2
 decorator==4.4.2
-Django==2.2.12
+Django==2.2.13
 django-cacheops==5.0
-django-cleanup==4.0.0
+django-cleanup==5.0.0
 django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-dynamic-fixture==3.1.0
-django-filter==2.2.0
+django-filter==2.3.0
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
 django-prometheus==2.0.0
@@ -51,16 +51,16 @@ et-xmlfile==1.0.1
 flake8==3.8.2
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.17.0
-google-auth==1.16.0
+google-api-core==1.19.1
+google-auth==1.16.1
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.9
-importlib-metadata==1.6.0
-inflection==0.4.0
+importlib-metadata==1.6.1
+inflection==0.5.0
 itypes==1.2.0
 jdcal==1.4.1
 Jinja2==2.11.2
@@ -89,7 +89,7 @@ pyrsistent==0.16.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
-redis==3.5.2
+redis==3.5.3
 requests==2.23.0
 rsa==4.0
 ruamel.yaml==0.16.10
@@ -100,6 +100,7 @@ six==1.15.0
 spavro==1.1.23
 sqlparse==0.3.1
 tblib==1.6.0
+toml==0.10.1
 uritemplate==3.0.1
 urllib3==1.25.9
 uWSGI==2.0.18
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index af37d31be..f6a4cfad3 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -12,20 +12,20 @@
 #
 ################################################################################
 
-aether.sdk==1.2.22
-autopep8==1.5.2
-boto3==1.13.19
-botocore==1.16.19
+aether.sdk==1.3.0
+autopep8==1.5.3
+boto3==1.13.24
+botocore==1.16.24
 cachetools==4.1.0
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
 coverage==5.1
 cryptography==2.9.2
-Django==2.2.12
+Django==2.2.13
 django-cacheops==5.0
-django-cleanup==4.0.0
+django-cleanup==5.0.0
 django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
@@ -41,15 +41,15 @@ flake8==3.8.2
 flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
-google-api-core==1.17.0
-google-auth==1.16.0
+google-api-core==1.19.1
+google-auth==1.16.1
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.9
-importlib-metadata==1.6.0
+importlib-metadata==1.6.1
 Jinja2==2.11.2
 jmespath==0.10.0
 linecache2==1.0.0
@@ -71,7 +71,7 @@ python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
 pyxform==1.1.0
-redis==3.5.2
+redis==3.5.3
 requests==2.23.0
 rsa==4.0
 s3transfer==0.3.3
@@ -80,6 +80,7 @@ six==1.15.0
 spavro==1.1.23
 sqlparse==0.3.1
 tblib==1.6.0
+toml==0.10.1
 traceback2==1.4.0
 unicodecsv==0.14.1
 unittest2==1.1.0
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index 9c58029ea..81384ca60 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -13,7 +13,7 @@
 ################################################################################
 
 attrs==19.3.0
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 cffi==1.14.0
 chardet==3.0.4
 click==7.1.2
@@ -22,10 +22,10 @@ cryptography==2.9.2
 flake8==3.8.2
 flake8-quotes==3.2.0
 Flask==1.1.2
-gevent==20.5.2
-greenlet==0.4.15
+gevent==20.6.0
+greenlet==0.4.16
 idna==2.9
-importlib-metadata==1.6.0
+importlib-metadata==1.6.1
 itsdangerous==1.1.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
@@ -41,13 +41,13 @@ pycparser==2.20
 pyflakes==2.2.0
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
-pytest==5.4.2
+pytest==5.4.3
 requests==2.23.0
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.17
 urllib3==1.25.9
-wcwidth==0.1.9
+wcwidth==0.2.4
 Werkzeug==1.0.1
 zipp==3.1.0
 zope.event==4.4
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index 256953753..d16a3a3c9 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -48,7 +48,7 @@
     "@hot-loader/react-dom": "~16.13.0",
     "babel-loader": "~8.1.0",
     "css-loader": "~3.5.0",
-    "eslint": "~7.1.0",
+    "eslint": "~7.2.0",
     "enzyme": "~3.11.0",
     "enzyme-adapter-react-16": "~1.15.0",
     "express": "~4.17.0",
@@ -62,7 +62,7 @@
     "sass-loader": "~8.0.0",
     "standard": "~14.3.0",
     "style-loader": "~1.2.0",
-    "stylelint": "~13.5.0",
+    "stylelint": "~13.6.0",
     "stylelint-config-standard": "~20.0.0",
     "webpack": "~4.43.0",
     "webpack-bundle-tracker": "~0.4.3",
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index b2bb59e9b..b44733666 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -12,20 +12,20 @@
 #
 ################################################################################
 
-aether.sdk==1.2.22
-autopep8==1.5.2
-boto3==1.13.19
-botocore==1.16.19
+aether.sdk==1.3.0
+autopep8==1.5.3
+boto3==1.13.24
+botocore==1.16.24
 cachetools==4.1.0
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
 coverage==5.1
 cryptography==2.9.2
-Django==2.2.12
+Django==2.2.13
 django-cacheops==5.0
-django-cleanup==4.0.0
+django-cleanup==5.0.0
 django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
@@ -42,15 +42,15 @@ drf-dynamic-fields==0.3.1
 flake8==3.8.2
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.17.0
-google-auth==1.16.0
+google-api-core==1.19.1
+google-auth==1.16.1
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.9
-importlib-metadata==1.6.0
+importlib-metadata==1.6.1
 Jinja2==2.11.2
 jmespath==0.10.0
 MarkupSafe==1.1.1
@@ -69,7 +69,7 @@ pyOpenSSL==19.1.0
 python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
-redis==3.5.2
+redis==3.5.3
 requests==2.23.0
 rsa==4.0
 s3transfer==0.3.3
@@ -77,6 +77,7 @@ sentry-sdk==0.14.4
 six==1.15.0
 sqlparse==0.3.1
 tblib==1.6.0
+toml==0.10.1
 urllib3==1.25.9
 uWSGI==2.0.18
 zipp==3.1.0
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index 1144ec4c0..f7a1640a9 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -13,7 +13,7 @@
 # See more in: https://docs.docker.com/compose/extends/
 # ------------------------------------------------------------------------------
 
-version: "2.1"
+version: "2.4"
 
 services:
 
diff --git a/docker-compose-connect.yml b/docker-compose-connect.yml
index b2e82146c..5f5b2029f 100644
--- a/docker-compose-connect.yml
+++ b/docker-compose-connect.yml
@@ -5,7 +5,7 @@
 #  * Aether Kafka Producer
 # ------------------------------------------------------------------------------
 
-version: "2.1"
+version: "2.4"
 
 networks:
   aether:
diff --git a/docker-compose-test.yml b/docker-compose-test.yml
index 006253d6b..3594a4b19 100644
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@@ -10,7 +10,7 @@
 #  * Aether Integration Tests
 # ------------------------------------------------------------------------------
 
-version: "2.1"
+version: "2.4"
 
 services:
 
diff --git a/docker-compose.yml b/docker-compose.yml
index 2ec038678..2a03e23d6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@
 #  * Aether UI & UI Assets
 # ------------------------------------------------------------------------------
 
-version: "2.1"
+version: "2.4"
 
 networks:
   # docker network create ${NETWORK_NAME}

From 1ce29b5c2b9ce63a37ef2e697cc90924aa164602 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Thu, 11 Jun 2020 17:05:55 +0200
Subject: [PATCH 10/29] chore: upgrade dependencies (#848)

---
 aether-kernel/conf/pip/requirements.txt     | 16 +++++++++-------
 aether-odk-module/conf/pip/requirements.txt | 16 +++++++++-------
 aether-producer/conf/pip/requirements.txt   |  4 ++--
 aether-ui/conf/pip/requirements.txt         | 16 +++++++++-------
 4 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 2c7a455d6..84144e776 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -13,11 +13,11 @@
 ################################################################################
 
 aether.python==1.0.17
-aether.sdk==1.3.0
+aether.sdk==1.3.1
 attrs==19.3.0
 autopep8==1.5.3
-boto3==1.13.24
-botocore==1.16.24
+boto3==1.14.0
+botocore==1.17.0
 cachetools==4.1.0
 certifi==2020.4.5.2
 cffi==1.14.0
@@ -37,6 +37,7 @@ django-dynamic-fixture==3.1.0
 django-filter==2.3.0
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
+django-postgrespool2==1.0.1
 django-prometheus==2.0.0
 django-redis==4.12.1
 django-silk==4.0.1
@@ -48,11 +49,11 @@ drf-dynamic-fields==0.3.1
 drf-yasg==1.17.1
 eha-jsonpath==0.5.1
 et-xmlfile==1.0.1
-flake8==3.8.2
+flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.19.1
-google-auth==1.16.1
+google-api-core==1.20.0
+google-auth==1.17.0
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
 google-resumable-media==0.5.1
@@ -91,13 +92,14 @@ python-json-logger==0.1.11
 pytz==2020.1
 redis==3.5.3
 requests==2.23.0
-rsa==4.0
+rsa==4.1
 ruamel.yaml==0.16.10
 ruamel.yaml.clib==0.2.0
 s3transfer==0.3.3
 sentry-sdk==0.14.4
 six==1.15.0
 spavro==1.1.23
+SQLAlchemy==1.3.17
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index f6a4cfad3..88ac34c68 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -12,10 +12,10 @@
 #
 ################################################################################
 
-aether.sdk==1.3.0
+aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.13.24
-botocore==1.16.24
+boto3==1.14.0
+botocore==1.17.0
 cachetools==4.1.0
 certifi==2020.4.5.2
 cffi==1.14.0
@@ -29,6 +29,7 @@ django-cleanup==5.0.0
 django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
+django-postgrespool2==1.0.1
 django-prometheus==2.0.0
 django-redis==4.12.1
 django-silk==4.0.1
@@ -37,12 +38,12 @@ django-uwsgi==0.2.2
 djangorestframework==3.11.0
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
-flake8==3.8.2
+flake8==3.8.3
 flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
-google-api-core==1.19.1
-google-auth==1.16.1
+google-api-core==1.20.0
+google-auth==1.17.0
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
 google-resumable-media==0.5.1
@@ -73,11 +74,12 @@ pytz==2020.1
 pyxform==1.1.0
 redis==3.5.3
 requests==2.23.0
-rsa==4.0
+rsa==4.1
 s3transfer==0.3.3
 sentry-sdk==0.14.4
 six==1.15.0
 spavro==1.1.23
+SQLAlchemy==1.3.17
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index 81384ca60..86353316a 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -19,10 +19,10 @@ chardet==3.0.4
 click==7.1.2
 confluent-kafka==1.4.2
 cryptography==2.9.2
-flake8==3.8.2
+flake8==3.8.3
 flake8-quotes==3.2.0
 Flask==1.1.2
-gevent==20.6.0
+gevent==20.6.1
 greenlet==0.4.16
 idna==2.9
 importlib-metadata==1.6.1
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index b44733666..3ee6f004d 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -12,10 +12,10 @@
 #
 ################################################################################
 
-aether.sdk==1.3.0
+aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.13.24
-botocore==1.16.24
+boto3==1.14.0
+botocore==1.17.0
 cachetools==4.1.0
 certifi==2020.4.5.2
 cffi==1.14.0
@@ -30,6 +30,7 @@ django-cors-headers==3.3.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
+django-postgrespool2==1.0.1
 django-prometheus==2.0.0
 django-redis==4.12.1
 django-silk==4.0.1
@@ -39,11 +40,11 @@ django-webpack-loader==0.7.0
 djangorestframework==3.11.0
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
-flake8==3.8.2
+flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.19.1
-google-auth==1.16.1
+google-api-core==1.20.0
+google-auth==1.17.0
 google-cloud-core==1.3.0
 google-cloud-storage==1.28.1
 google-resumable-media==0.5.1
@@ -71,10 +72,11 @@ python-json-logger==0.1.11
 pytz==2020.1
 redis==3.5.3
 requests==2.23.0
-rsa==4.0
+rsa==4.1
 s3transfer==0.3.3
 sentry-sdk==0.14.4
 six==1.15.0
+SQLAlchemy==1.3.17
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1

From 83f9bfc6b87a684db99bebe6588ae9dc21ad5f73 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 15 Jun 2020 10:14:58 +0200
Subject: [PATCH 11/29] fix(kernel): remove silk workaround (#850)

---
 aether-kernel/aether/kernel/settings.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/aether-kernel/aether/kernel/settings.py b/aether-kernel/aether/kernel/settings.py
index 07b47369c..c05c34141 100644
--- a/aether-kernel/aether/kernel/settings.py
+++ b/aether-kernel/aether/kernel/settings.py
@@ -25,7 +25,6 @@
 from aether.sdk.conf.settings_aether import (
     INSTALLED_APPS,
     MIGRATION_MODULES,
-    PROFILING_ENABLED,
     REST_FRAMEWORK,
 )
 
@@ -84,20 +83,6 @@
     EXPORT_NUM_CHUNKS = 1
 
 
-# Profiling workaround
-# ------------------------------------------------------------------------------
-#
-# Issue: The entities bulk creation is failing with Silk enabled.
-# The reported bug, https://github.com/jazzband/django-silk/issues/348,
-# In the meantime we will disable silk for those requests.
-
-if PROFILING_ENABLED:
-    def ignore_entities_post(request):
-        return request.method != 'POST' or '/entities' not in request.path
-
-    SILKY_INTERCEPT_FUNC = ignore_entities_post
-
-
 # Swagger workaround
 # ------------------------------------------------------------------------------
 # The ``bravado`` lib in ``aether.client`` cannot deal with JSON fields handled

From fbea9c7489a635fe47c1c9c59d98783e75c59823 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 15 Jun 2020 10:47:56 +0200
Subject: [PATCH 12/29] feat: performance tests (#849)

---
 docker-compose-locust.yml           |  50 ++++++++++++
 scripts/build_docker_credentials.sh |   1 +
 tests/performance/kernel_taskset.py | 118 ++++++++++++++++++++++++++++
 tests/performance/locustfile.py     |  31 ++++++++
 tests/performance/settings.py       |  25 ++++++
 5 files changed, 225 insertions(+)
 create mode 100644 docker-compose-locust.yml
 create mode 100644 tests/performance/kernel_taskset.py
 create mode 100644 tests/performance/locustfile.py
 create mode 100644 tests/performance/settings.py

diff --git a/docker-compose-locust.yml b/docker-compose-locust.yml
new file mode 100644
index 000000000..04f76a240
--- /dev/null
+++ b/docker-compose-locust.yml
@@ -0,0 +1,50 @@
+# ---------------------------------------------------------------------------- #
+#                        Performance Tests
+#
+# Steps:
+#
+# 1. Start the app as usual.
+#
+# 2. Run:    docker-compose -f docker-compose-locust.yml up --quiet-pull
+#
+# 3. Go to:  http://localhost:8089
+#
+# 4. Indicate number of total users and hatch rate (>0) and press "start".
+#
+# ---------------------------------------------------------------------------- #
+
+version: "2.4"
+
+networks:
+  internal:
+    external:
+      name: ${NETWORK_NAME}
+
+
+services:
+  locust-master:
+    image: locustio/locust
+    environment: &locust_env
+      BASE_HOST: http://${NETWORK_DOMAIN}
+      AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
+      AETHER_KERNEL_URL: http://${NETWORK_DOMAIN}/kernel
+    volumes: &locust_volumes
+      - ./tests/performance:/mnt/locust
+    ports:
+      - 8089:8089
+    command: -f /mnt/locust/locustfile.py --master
+    networks:
+      - internal
+    extra_hosts:
+      - ${NETWORK_DOMAIN}:${NETWORK_NGINX_IP}
+
+  locust-worker:
+    image: locustio/locust
+    environment: *locust_env
+    volumes: *locust_volumes
+    scale: ${TEST_WORKERS:-5}
+    command: -f /mnt/locust/locustfile.py --worker --master-host locust-master
+    networks:
+      - internal
+    extra_hosts:
+      - ${NETWORK_DOMAIN}:${NETWORK_NGINX_IP}
diff --git a/scripts/build_docker_credentials.sh b/scripts/build_docker_credentials.sh
index 5f865286c..817bdb1d2 100755
--- a/scripts/build_docker_credentials.sh
+++ b/scripts/build_docker_credentials.sh
@@ -157,6 +157,7 @@ CLIENT_REALM=test
 # ==================================================================
 # set to 1 to disable parallel execution
 TEST_PARALLEL=
+TEST_WORKERS=5
 
 # to speed up development changes in the SDK library
 # https://github.com/eHealthAfrica/aether-django-sdk-library
diff --git a/tests/performance/kernel_taskset.py b/tests/performance/kernel_taskset.py
new file mode 100644
index 000000000..058b448a7
--- /dev/null
+++ b/tests/performance/kernel_taskset.py
@@ -0,0 +1,118 @@
+# Copyright (C) 2020 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import random
+import uuid
+
+from locust import TaskSet, task
+
+from settings import AETHER_KERNEL_URL, AETHER_AUTH_HEADER
+
+
+class KernelTaskSet(TaskSet):
+
+    def on_start(self):
+        response = self.client.get(
+            name='/',
+            headers=AETHER_AUTH_HEADER,
+            url=f'{AETHER_KERNEL_URL}/',
+        )
+
+        # create initial project
+        self.create_avro_schemas()
+
+    @task(1)
+    def health_page(self):
+        response = self.client.get(
+            name='/health',
+            url=f'{AETHER_KERNEL_URL}/health',
+        )
+
+    @task(5)
+    def view_projects(self):
+        response = self.client.get(
+            name='/projects',
+            headers=AETHER_AUTH_HEADER,
+            url=f'{AETHER_KERNEL_URL}/projects.json',
+        )
+
+    @task(2)
+    def create_avro_schemas(self):
+        project_id = str(uuid.uuid4())
+        avro_schema = {
+            'name': f'simple-{project_id}',
+            'type': 'record',
+            'fields': [
+                {
+                'name': 'id',
+                'type': 'string',
+                },
+                {
+                'name': 'name',
+                'type': 'string',
+                }
+            ],
+        }
+
+        response = self.client.request(
+            name='/projects/avro-schemas',
+            headers=AETHER_AUTH_HEADER,
+            method='PATCH',
+            url=f'{AETHER_KERNEL_URL}/projects/{project_id}/avro-schemas.json',
+            json={
+                'name': str(project_id),
+                'avro_schemas': [{'definition': avro_schema}],
+            },
+        )
+
+    @task(15)
+    def create_submission(self):
+        # get list of mapping set ids
+        response = self.client.get(
+            url=f'{AETHER_KERNEL_URL}/mappingsets.json?fields=id&page_size=100',
+            name='/mappingsets',
+            headers=AETHER_AUTH_HEADER,
+        )
+        response.raise_for_status()
+        data = response.json()
+        if data['count'] == 0:
+            return
+
+        # choose one random mapping set id
+        results = data['results']
+        size = len(results)
+        _index = random.randint(0, size - 1)
+        mappingset_id = results[_index]['id']
+
+        submission_id = str(uuid.uuid4())
+        submission_payload = {
+            'id': submission_id,
+            'name': f'Name {submission_id}',
+        }
+
+        response = self.client.request(
+            name='/submissions',
+            headers=AETHER_AUTH_HEADER,
+            method='POST',
+            url=f'{AETHER_KERNEL_URL}/submissions.json',
+            json={
+                'id': submission_id,
+                'mappingset': mappingset_id,
+                'payload': submission_payload,
+            },
+        )
diff --git a/tests/performance/locustfile.py b/tests/performance/locustfile.py
new file mode 100644
index 000000000..a57ebf6a5
--- /dev/null
+++ b/tests/performance/locustfile.py
@@ -0,0 +1,31 @@
+# Copyright (C) 2020 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from locust import HttpUser, between
+
+from kernel_taskset import KernelTaskSet
+from settings import BASE_HOST
+
+
+class KernelUser(HttpUser):
+
+    host = BASE_HOST
+    tasks = [KernelTaskSet]
+
+    # wating time (in seconds) between two consecutive tasks
+    wait_time = between(0.1, 1)
diff --git a/tests/performance/settings.py b/tests/performance/settings.py
new file mode 100644
index 000000000..c7b83d971
--- /dev/null
+++ b/tests/performance/settings.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2020 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+BASE_HOST = os.environ['BASE_HOST']
+
+AETHER_KERNEL_URL = os.environ['AETHER_KERNEL_URL']
+AETHER_KERNEL_TOKEN = os.environ['AETHER_KERNEL_TOKEN']
+AETHER_AUTH_HEADER = {'Authorization': f'Token {AETHER_KERNEL_TOKEN}'}

From 29ea0c3f3ed3dbc929d0eaf677805060529ca571 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 15 Jun 2020 10:50:29 +0200
Subject: [PATCH 13/29] chore: wrong internal AETHER_KERNEL_URL

---
 docker-compose-base.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index f7a1640a9..3b25c3363 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -339,7 +339,7 @@ services:
 
       # These variables will override the ones indicated in the settings file
       AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
-      AETHER_KERNEL_URL: http://kernel:8100
+      AETHER_KERNEL_URL: http://kernel:8100/kernel
       DEFAULT_REALM: ${DEFAULT_REALM}
       REALM_COOKIE: ${REALM_COOKIE}
 

From 65cf97a7a2df9fe424411d771b239dbccd76d917 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Tue, 16 Jun 2020 08:27:33 +0200
Subject: [PATCH 14/29] test: assertRaises check exception outside with (#854)

---
 .../aether/kernel/api/tests/test_models.py    | 24 ++++++------
 .../kernel/api/tests/test_serializers.py      | 23 +++++------
 .../aether/odk/api/tests/test_kernel_utils.py | 14 +++----
 .../aether/odk/api/tests/test_models.py       |  4 +-
 .../aether/odk/api/tests/test_serializers.py  |  2 -
 .../aether/odk/api/tests/test_xform_utils.py  | 38 +++++++++----------
 .../aether/ui/api/tests/test_serializers.py   |  4 +-
 aether-ui/aether/ui/api/tests/test_utils.py   | 14 +++----
 8 files changed, 59 insertions(+), 64 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/tests/test_models.py b/aether-kernel/aether/kernel/api/tests/test_models.py
index 68f060e2e..a08f73a65 100644
--- a/aether-kernel/aether/kernel/api/tests/test_models.py
+++ b/aether-kernel/aether/kernel/api/tests/test_models.py
@@ -70,7 +70,7 @@ def test_models(self):
         self.assertFalse(schemadecorator.is_accessible(REALM))
         self.assertIsNone(schemadecorator.get_realm())
 
-        with self.assertRaises(IntegrityError) as err:
+        with self.assertRaises(IntegrityError) as err1:
             models.MappingSet.objects.create(
                 revision='a sample revision',
                 name='a sample mapping set',
@@ -78,9 +78,9 @@ def test_models(self):
                 input=EXAMPLE_SOURCE_DATA,
                 project=project,
             )
-            self.assertIn('Input does not conform to schema', str(err.exception))
+        self.assertIn('Input does not conform to schema', str(err1.exception))
 
-        with self.assertRaises(IntegrityError) as err:
+        with self.assertRaises(IntegrityError) as err2:
             models.MappingSet.objects.create(
                 revision='a sample revision',
                 name='a sample mapping set',
@@ -91,7 +91,7 @@ def test_models(self):
                 input={},
                 project=project,
             )
-            self.assertIn('Record schema requires a non-empty fields property.', str(err.exception))
+        self.assertIn('Record schema requires a non-empty fields property.', str(err2.exception))
 
         mappingset = models.MappingSet.objects.create(
             revision='a sample revision',
@@ -136,9 +136,9 @@ def test_models(self):
 
         # check mapping definition validation
         mapping.definition = {'entities': {'a': str(uuid.uuid4())}}
-        with self.assertRaises(IntegrityError) as err:
+        with self.assertRaises(IntegrityError) as err3:
             mapping.save()
-            self.assertIn('is not valid under any of the given schemas', str(err.exception))
+        self.assertIn('is not valid under any of the given schemas', str(err3.exception))
 
         submission = models.Submission.objects.create(
             revision='a sample revision',
@@ -184,15 +184,15 @@ def test_models(self):
         self.assertEqual(attachment_2.submission_revision, 'next revision')
         self.assertNotEqual(attachment_2.submission_revision, submission.revision)
 
-        with self.assertRaises(IntegrityError) as err:
+        with self.assertRaises(IntegrityError) as err4:
             models.Entity.objects.create(
                 revision='a sample revision',
                 payload=EXAMPLE_SOURCE_DATA,  # this is the submission payload without ID
                 status='Publishable',
                 schemadecorator=schemadecorator,
             )
-            self.assertIn('Extracted record did not conform to registered schema',
-                          str(err.exception))
+        self.assertIn('Extracted record did not conform to registered schema',
+                      str(err4.exception))
 
         entity = models.Entity.objects.create(
             revision='a sample revision',
@@ -227,8 +227,8 @@ def test_models(self):
         entity.schemadecorator = schemadecorator_2
         with self.assertRaises(IntegrityError) as ie:
             entity.save()
-            self.assertIn('Submission, Mapping and Schema Decorator MUST belong to the same Project',
-                          str(ie.exception))
+        self.assertIn('Submission, Mapping and Schema Decorator MUST belong to the same Project',
+                      str(ie.exception))
 
         # it works without submission
         entity.submission = None
@@ -305,7 +305,7 @@ def test_models_ids(self):
         # it's trying to create a new schema (not replacing the current one)
         with self.assertRaises(IntegrityError) as ie:
             schema.save()
-            self.assertIn('Key (name)=(a first schema name) already exists.', str(ie.exception))
+        self.assertIn('Key (name)=(a first schema name) already exists.', str(ie.exception))
 
         # if we change the name we'll end up with two schemas
         schema.name = 'a second schema name'
diff --git a/aether-kernel/aether/kernel/api/tests/test_serializers.py b/aether-kernel/aether/kernel/api/tests/test_serializers.py
index 176d11b1a..845059654 100644
--- a/aether-kernel/aether/kernel/api/tests/test_serializers.py
+++ b/aether-kernel/aether/kernel/api/tests/test_serializers.py
@@ -180,10 +180,10 @@ def test__serializers__create_and_update(self):
         )
         self.assertTrue(submission.is_valid(), submission.errors)
 
-        with self.assertRaises(ValidationError) as ve:
+        with self.assertRaises(ValidationError) as ve_s:
             submission.save()
-            self.assertIn('Mapping set must be provided on initial submission',
-                          str(ve.exception))
+        self.assertIn('Mapping set must be provided on initial submission',
+                      str(ve_s.exception))
 
         # check the submission with entity extraction errors
         submission = serializers.SubmissionSerializer(
@@ -233,10 +233,10 @@ def test__serializers__create_and_update(self):
         )
         self.assertTrue(entity.is_valid(), entity.errors)
 
-        with self.assertRaises(Exception) as ve:
+        with self.assertRaises(Exception) as ve_e:
             entity.save()
             self.assertIn('Extracted record did not conform to registered schema',
-                          str(ve.exception))
+                          str(ve_e.exception))
 
         # create entity
         entity_2 = serializers.EntitySerializer(
@@ -267,8 +267,8 @@ def test__serializers__create_and_update(self):
 
         with self.assertRaises(Exception) as ve_3:
             entity_3.save()
-            self.assertIn('Extracted record did not conform to registered schema',
-                          str(ve_3.exception))
+        self.assertIn('Extracted record did not conform to registered schema',
+                      str(ve_3.exception))
 
         entity_4 = serializers.EntitySerializer(
             models.Entity.objects.get(pk=entity_2.data['id']),
@@ -297,13 +297,10 @@ def test__serializers__create_and_update(self):
             context={'request': self.request},
         )
         self.assertTrue(bad_bulk.is_valid(), bad_bulk.errors)
-        try:
+        with self.assertRaises(Exception) as ve_be:
             bad_bulk.save()
-            self.assertTrue(False)  # This should have raised a ValidationError
-        except ValidationError:
-            self.assertTrue(True)
-        except Exception:
-            self.assertTrue(False)  # This should have raised a ValidationError
+        self.assertIn('Extracted record did not conform to registered schema',
+                      str(ve_be.exception))
 
         # good bulk
 
diff --git a/aether-odk-module/aether/odk/api/tests/test_kernel_utils.py b/aether-odk-module/aether/odk/api/tests/test_kernel_utils.py
index 236b28687..f52b6c232 100644
--- a/aether-odk-module/aether/odk/api/tests/test_kernel_utils.py
+++ b/aether-odk-module/aether/odk/api/tests/test_kernel_utils.py
@@ -98,8 +98,8 @@ def test__upsert_kernel_artefacts__no_connection(self, mock_auth, mock_patch):
                 artefacts={'avro_schemas': []},
             )
 
-            self.assertIn('Connection with Aether Kernel server is not possible.',
-                          str(kpe.exception), kpe)
+        self.assertIn('Connection with Aether Kernel server is not possible.',
+                      str(kpe.exception), kpe)
 
         mock_auth.assert_called_once()
         mock_patch.assert_not_called()
@@ -115,11 +115,11 @@ def test__upsert_kernel_artefacts__unexpected_error(self, mock_auth, mock_patch)
                 artefacts={'avro_schemas': []},
             )
 
-            self.assertIn('Unexpected response from Aether Kernel server',
-                          str(kpe.exception), kpe)
-            self.assertIn('while trying to create/update the project artefacts',
-                          str(kpe.exception), kpe)
-            self.assertIn(f'"{str(self.project.project_id)}"', str(kpe.exception), kpe)
+        self.assertIn('Unexpected response from Aether Kernel server',
+                      str(kpe.exception), kpe)
+        self.assertIn('while trying to create/update the project artefacts',
+                      str(kpe.exception), kpe)
+        self.assertIn(f'"{str(self.project.project_id)}"', str(kpe.exception), kpe)
 
         mock_auth.assert_called_once()
         mock_patch.assert_called_once_with(
diff --git a/aether-odk-module/aether/odk/api/tests/test_models.py b/aether-odk-module/aether/odk/api/tests/test_models.py
index 27e0777cf..9d590a276 100644
--- a/aether-odk-module/aether/odk/api/tests/test_models.py
+++ b/aether-odk-module/aether/odk/api/tests/test_models.py
@@ -185,8 +185,8 @@ def test__xform__uniqueness(self):
                 project=project,
                 xml_data=self.samples['xform']['xml-ok'],
             )
-            self.assertIn('duplicate key value violates unique constraint',
-                          str(ie.exception))
+        self.assertIn('Xform with this Project, XForm ID and XForm version already exists.',
+                      str(ie.exception))
 
         # it works with another version
         XForm.objects.create(
diff --git a/aether-odk-module/aether/odk/api/tests/test_serializers.py b/aether-odk-module/aether/odk/api/tests/test_serializers.py
index 5f5b8cc43..d829c915b 100644
--- a/aether-odk-module/aether/odk/api/tests/test_serializers.py
+++ b/aether-odk-module/aether/odk/api/tests/test_serializers.py
@@ -203,8 +203,6 @@ def test_surveyor_serializer__no_password(self):
         self.assertTrue(user.is_valid(), user.errors)
         with self.assertRaises(ValidationError) as ve:
             user.save()
-
-        self.assertIsNotNone(ve)
         self.assertIn('This field is required.', str(ve.exception), ve)
 
     def test_surveyor_serializer__empty_password(self):
diff --git a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
index 16473d7e9..bfee64c73 100644
--- a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
+++ b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
@@ -44,7 +44,7 @@ class XFormUtilsValidatorsTests(CustomTestCase):
     def test__validate_xform__not_valid(self):
         with self.assertRaises(XFormParseError) as ve:
             validate_xform(self.samples['xform']['xml-err'])
-            self.assertIn('Not valid xForm definition.', str(ve.exception), ve)
+        self.assertIn('Not valid xForm definition.', str(ve.exception), ve)
 
     def test__validate_xform__missing_required__html(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -53,8 +53,8 @@ def test__validate_xform__missing_required__html(self):
                     <html></html>
                 '''
             )
-            self.assertIn('Missing required tags:', str(ve.exception), ve)
-            self.assertIn('<h:html>', str(ve.exception), ve)
+        self.assertIn('Missing required tags:', str(ve.exception), ve)
+        self.assertIn('<h:html>', str(ve.exception), ve)
 
     def test__validate_xform__missing_required__html__children(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -66,9 +66,9 @@ def test__validate_xform__missing_required__html__children(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required tags:', str(ve.exception), ve)
-            self.assertIn('<h:body> in <h:html>', str(ve.exception), ve)
-            self.assertIn('<h:head> in <h:html>', str(ve.exception), ve)
+        self.assertIn('Missing required tags:', str(ve.exception), ve)
+        self.assertIn('<h:body> in <h:html>', str(ve.exception), ve)
+        self.assertIn('<h:head> in <h:html>', str(ve.exception), ve)
 
     def test__validate_xform__missing_required__head__children(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -83,9 +83,9 @@ def test__validate_xform__missing_required__head__children(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required tags:', str(ve.exception), ve)
-            self.assertIn('<h:title> in <h:html><h:head>', str(ve.exception), ve)
-            self.assertIn('<model> in <h:html><h:head>', str(ve.exception), ve)
+        self.assertIn('Missing required tags:', str(ve.exception), ve)
+        self.assertIn('<h:title> in <h:html><h:head>', str(ve.exception), ve)
+        self.assertIn('<model> in <h:html><h:head>', str(ve.exception), ve)
 
     def test__validate_xform__missing_required__model__children(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -102,8 +102,8 @@ def test__validate_xform__missing_required__model__children(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required tags:', str(ve.exception), ve)
-            self.assertIn('<instance> in <h:html><h:head><model>', str(ve.exception), ve)
+        self.assertIn('Missing required tags:', str(ve.exception), ve)
+        self.assertIn('<instance> in <h:html><h:head><model>', str(ve.exception), ve)
 
     def test__validate_xform__no_instance(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -123,7 +123,7 @@ def test__validate_xform__no_instance(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required instance definition.', str(ve.exception), ve)
+        self.assertIn('Missing required instance definition.', str(ve.exception), ve)
 
     def test__validate_xform__no_title__no_form_id(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -144,7 +144,7 @@ def test__validate_xform__no_title__no_form_id(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required form title and instance ID.', str(ve.exception), ve)
+        self.assertIn('Missing required form title and instance ID.', str(ve.exception), ve)
 
     def test__validate_xform__no_title__blank(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -165,7 +165,7 @@ def test__validate_xform__no_title__blank(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required form title.', str(ve.exception), ve)
+        self.assertIn('Missing required form title.', str(ve.exception), ve)
 
     def test__validate_xform__no_xform_id(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -186,7 +186,7 @@ def test__validate_xform__no_xform_id(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required instance ID.', str(ve.exception), ve)
+        self.assertIn('Missing required instance ID.', str(ve.exception), ve)
 
     def test__validate_xform__no_xform_id__blank(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -207,7 +207,7 @@ def test__validate_xform__no_xform_id__blank(self):
                     </h:html>
                 '''
             )
-            self.assertIn('Missing required instance ID.', str(ve.exception), ve)
+        self.assertIn('Missing required instance ID.', str(ve.exception), ve)
 
     def test__validate_xform__with__title__and__xform_id(self):
         try:
@@ -373,7 +373,7 @@ def test__get_avro_type__required(self):
     def test__get_xform_instance__error(self):
         with self.assertRaises(XFormParseError) as ve:
             get_instance({})
-            self.assertIn('Missing required instance definition.', str(ve.exception), ve)
+        self.assertIn('Missing required instance definition.', str(ve.exception), ve)
 
     def test__get_xform_instance__error__no_instances(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -386,7 +386,7 @@ def test__get_xform_instance__error__no_instances(self):
                     }
                 }
             })
-            self.assertIn('Missing required instance definition.', str(ve.exception), ve)
+        self.assertIn('Missing required instance definition.', str(ve.exception), ve)
 
     def test__get_xform_instance__error___no_default_instance(self):
         with self.assertRaises(XFormParseError) as ve:
@@ -403,7 +403,7 @@ def test__get_xform_instance__error___no_default_instance(self):
                     }
                 }
             })
-            self.assertIn('Missing required instance definition.', str(ve.exception), ve)
+        self.assertIn('Missing required instance definition.', str(ve.exception), ve)
 
     def test__get_xform_instance(self):
         xform_dict = {
diff --git a/aether-ui/aether/ui/api/tests/test_serializers.py b/aether-ui/aether/ui/api/tests/test_serializers.py
index 5bc20cecb..cefe4255f 100644
--- a/aether-ui/aether/ui/api/tests/test_serializers.py
+++ b/aether-ui/aether/ui/api/tests/test_serializers.py
@@ -126,7 +126,7 @@ def test__serializers__create_and_update(self):
 
         with self.assertRaises(Exception) as ve_c:
             contract_3.save()
-            self.assertIn('Contract is read only', str(ve_c.exception))
+        self.assertIn('Contract is read only', str(ve_c.exception))
 
         # try to update the pipeline
         pipeline_3 = serializers.PipelineSerializer(
@@ -141,4 +141,4 @@ def test__serializers__create_and_update(self):
 
         with self.assertRaises(Exception) as ve_p:
             pipeline_3.save()
-            self.assertIn('Pipeline is read only', str(ve_p.exception))
+        self.assertIn('Pipeline is read only', str(ve_p.exception))
diff --git a/aether-ui/aether/ui/api/tests/test_utils.py b/aether-ui/aether/ui/api/tests/test_utils.py
index 537d94b80..dc36dfea6 100644
--- a/aether-ui/aether/ui/api/tests/test_utils.py
+++ b/aether-ui/aether/ui/api/tests/test_utils.py
@@ -115,7 +115,7 @@ def test_publish(self):
         project_id = str(project.pk)
 
         # publish with exceptions
-        with self.assertRaises(utils.PublishError) as pe:
+        with self.assertRaises(utils.PublishError) as pe1:
             with mock.patch('aether.ui.api.utils.kernel_data_request',
                             side_effect=Exception('Error in project')) as mock_kernel:
                 utils.publish_project(project)
@@ -125,9 +125,9 @@ def test_publish(self):
                     data=mock.ANY,
                     headers={'Authorization': mock.ANY},
                 )
-            self.assertIn('Error in project', str(pe.exception))
+        self.assertIn('Error in project', str(pe1.exception))
 
-        with self.assertRaises(utils.PublishError) as pe:
+        with self.assertRaises(utils.PublishError) as pe2:
             with mock.patch('aether.ui.api.utils.kernel_data_request',
                             side_effect=Exception('Error in pipeline')) as mock_kernel:
                 utils.publish_pipeline(pipeline)
@@ -137,9 +137,9 @@ def test_publish(self):
                     data=mock.ANY,
                     headers={'Authorization': mock.ANY},
                 )
-            self.assertIn('Error in pipeline', str(pe.exception))
+        self.assertIn('Error in pipeline', str(pe2.exception))
 
-        with self.assertRaises(utils.PublishError) as pe:
+        with self.assertRaises(utils.PublishError) as pe3:
             with mock.patch('aether.ui.api.utils.kernel_data_request',
                             side_effect=Exception('Error in contract')) as mock_kernel:
                 with mock.patch('aether.ui.api.utils.publish_preflight',
@@ -152,7 +152,7 @@ def test_publish(self):
                         data=mock.ANY,
                         headers={'Authorization': mock.ANY},
                     )
-            self.assertIn('Error in contract', str(pe.exception))
+        self.assertIn('Error in contract', str(pe3.exception))
 
         # publish without exceptions
         with mock.patch('aether.ui.api.utils.kernel_data_request') as mock_kernel:
@@ -353,7 +353,7 @@ def test__kernel_workflow(self):
         contract.refresh_from_db()
         with self.assertRaises(utils.PublishError) as pe:
             utils.publish_contract(contract)
-            self.assertIn('Contract is read only', str(pe.exception))
+        self.assertIn('Contract is read only', str(pe.exception))
 
         contract.is_read_only = False
         contract.save()

From ca60bf726e323cef91ae56469d94291e6c6e10ab Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Tue, 16 Jun 2020 13:05:46 +0200
Subject: [PATCH 15/29] feat(kernel): input from AVRO schema endpoint (#857)

---
 .../aether/kernel/api/tests/test_views.py     | 40 +++++++++++++++++++
 aether-kernel/aether/kernel/api/urls.py       |  1 +
 aether-kernel/aether/kernel/api/views.py      | 27 ++++++++++++-
 3 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py
index f1f1ad11b..49ad21ec6 100644
--- a/aether-kernel/aether/kernel/api/tests/test_views.py
+++ b/aether-kernel/aether/kernel/api/tests/test_views.py
@@ -1049,3 +1049,43 @@ def test_update_by_filters(self):
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertEqual('No values to update', response.json())
+
+    def test__generate_avro_input(self):
+        url = reverse('generate-avro-input')
+
+        # no data
+        response = self.client.post(url)
+        self.assertEqual(response.status_code, 400)
+        data = json.loads(response.content)
+        self.assertEqual(data['message'], 'Missing "schema" data')
+
+        # from schema to input
+        schema = {
+            'name': 'Dummy',
+            'type': 'record',
+            'fields': [
+                {
+                    'name': 'id',
+                    'type': 'string'
+                },
+                {
+                    'name': 'name',
+                    'type': 'string'
+                },
+            ],
+        }
+        response = self.client.post(
+            url,
+            data=json.dumps({'schema': schema}),
+            content_type='application/json',
+        )
+        self.assertEqual(response.status_code, 200)
+        data = json.loads(response.content)
+
+        self.assertEqual(data['schema'], schema)
+        # input conforms the schema
+        self.assertIn('input', data)
+        self.assertIn('id', data['input'])
+        self.assertTrue(isinstance(data['input']['id'], str))
+        self.assertIn('name', data['input'])
+        self.assertTrue(isinstance(data['input']['name'], str))
diff --git a/aether-kernel/aether/kernel/api/urls.py b/aether-kernel/aether/kernel/api/urls.py
index 6710869f9..cb7e3fa08 100644
--- a/aether-kernel/aether/kernel/api/urls.py
+++ b/aether-kernel/aether/kernel/api/urls.py
@@ -39,4 +39,5 @@
 
 urlpatterns = router.urls + [
     path('validate-mappings/', view=views.validate_mappings_view, name='validate-mappings'),
+    path('generate-avro-input/', view=views.generate_avro_input, name='generate-avro-input'),
 ]
diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py
index d82b67c86..c3185c60b 100644
--- a/aether-kernel/aether/kernel/api/views.py
+++ b/aether-kernel/aether/kernel/api/views.py
@@ -35,11 +35,12 @@
 
 from aether.sdk.multitenancy.views import MtViewSetMixin
 from aether.sdk.drf.views import FilteredMixin
+from aether.python.avro.tools import random_avro, extract_jsonpaths_and_docs
 from aether.python.entity.extractor import (
     extract_create_entities,
     ENTITY_EXTRACTION_ERRORS,
 )
-from aether.python.avro.tools import extract_jsonpaths_and_docs
+
 from .constants import LINKED_DATA_MAX_DEPTH
 from .entity_extractor import run_entity_extraction
 from .exporter import ExporterMixin
@@ -849,3 +850,27 @@ def run_mapping_validation(submission_payload, mapping_definition, schemas):
         })
     except Exception as e:
         return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+@api_view(['POST'])
+@renderer_classes([JSONRenderer])
+@permission_classes([permissions.IsAuthenticated])
+def generate_avro_input(request, *args, **kwargs):
+    '''
+    Given a `schema` returns an input sample that conforms that schema.
+    '''
+
+    if 'schema' in request.data:
+        schema = request.data['schema']
+        input_sample = random_avro(schema)
+
+        return Response({
+            'schema': schema,
+            'input': input_sample,
+        })
+
+    else:
+        return Response(
+            data={'message': _('Missing "schema" data')},
+            status=status.HTTP_400_BAD_REQUEST,
+        )

From 51b0468b4077d8460c0420761678fa92d7d8017a Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 24 Jun 2020 09:00:19 +0200
Subject: [PATCH 16/29] chore: upgrade to python 3.8 (#855)

* chore: python 3.8

* chore: upgrade dependencies
---
 aether-client-library/Dockerfile              |  2 +-
 aether-kernel/Dockerfile                      |  2 +-
 .../conf/pip/primary-requirements.txt         |  1 +
 aether-kernel/conf/pip/requirements.txt       | 22 ++++++++---------
 aether-odk-module/Dockerfile                  |  2 +-
 .../conf/pip/primary-requirements.txt         |  1 +
 aether-odk-module/conf/pip/requirements.txt   | 24 +++++++++----------
 aether-producer/Dockerfile                    |  2 +-
 aether-producer/conf/pip/requirements.txt     | 14 +++++------
 aether-ui/Dockerfile                          |  2 +-
 aether-ui/aether/ui/assets/package.json       | 12 +++++-----
 aether-ui/conf/pip/primary-requirements.txt   |  1 +
 aether-ui/conf/pip/requirements.txt           | 24 +++++++++----------
 scripts/deployment/kernel.Dockerfile          |  2 +-
 scripts/deployment/odk.Dockerfile             |  2 +-
 scripts/deployment/producer.Dockerfile        |  2 +-
 scripts/deployment/ui.Dockerfile              |  2 +-
 test-aether-integration-module/Dockerfile     |  2 +-
 18 files changed, 58 insertions(+), 61 deletions(-)

diff --git a/aether-client-library/Dockerfile b/aether-client-library/Dockerfile
index 093693aea..e43ed1c49 100644
--- a/aether-client-library/Dockerfile
+++ b/aether-client-library/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 ARG VERSION=0.0.0
 
diff --git a/aether-kernel/Dockerfile b/aether-kernel/Dockerfile
index 30be24ddc..7a15b749b 100644
--- a/aether-kernel/Dockerfile
+++ b/aether-kernel/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Kernel" \
       name="aether-kernel" \
diff --git a/aether-kernel/conf/pip/primary-requirements.txt b/aether-kernel/conf/pip/primary-requirements.txt
index f2bfa3fec..4d4eaa1f7 100644
--- a/aether-kernel/conf/pip/primary-requirements.txt
+++ b/aether-kernel/conf/pip/primary-requirements.txt
@@ -23,6 +23,7 @@ aether.python
 
 
 # Django specific
+django<3
 django-filter
 django-model-utils
 drf-yasg
diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 84144e776..0fe11af6f 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -16,10 +16,10 @@ aether.python==1.0.17
 aether.sdk==1.3.1
 attrs==19.3.0
 autopep8==1.5.3
-boto3==1.14.0
-botocore==1.17.0
+boto3==1.14.9
+botocore==1.17.9
 cachetools==4.1.0
-certifi==2020.4.5.2
+certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
@@ -31,7 +31,7 @@ decorator==4.4.2
 Django==2.2.13
 django-cacheops==5.0
 django-cleanup==5.0.0
-django-cors-headers==3.3.0
+django-cors-headers==3.4.0
 django-debug-toolbar==2.2
 django-dynamic-fixture==3.1.0
 django-filter==2.3.0
@@ -52,10 +52,10 @@ et-xmlfile==1.0.1
 flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.20.0
-google-auth==1.17.0
+google-api-core==1.21.0
+google-auth==1.18.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.1
+google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
@@ -91,12 +91,12 @@ python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
 redis==3.5.3
-requests==2.23.0
-rsa==4.1
+requests==2.24.0
+rsa==4.6
 ruamel.yaml==0.16.10
 ruamel.yaml.clib==0.2.0
 s3transfer==0.3.3
-sentry-sdk==0.14.4
+sentry-sdk==0.15.1
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.17
@@ -105,5 +105,5 @@ tblib==1.6.0
 toml==0.10.1
 uritemplate==3.0.1
 urllib3==1.25.9
-uWSGI==2.0.18
+uWSGI==2.0.19.1
 zipp==3.1.0
diff --git a/aether-odk-module/Dockerfile b/aether-odk-module/Dockerfile
index 520733665..2dc313acc 100644
--- a/aether-odk-module/Dockerfile
+++ b/aether-odk-module/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether ODK Module" \
       name="aether-odk" \
diff --git a/aether-odk-module/conf/pip/primary-requirements.txt b/aether-odk-module/conf/pip/primary-requirements.txt
index 310f6ce68..b11e886e0 100644
--- a/aether-odk-module/conf/pip/primary-requirements.txt
+++ b/aether-odk-module/conf/pip/primary-requirements.txt
@@ -17,6 +17,7 @@
 
 # Aether Django SDK library with extras
 aether.sdk[cache,server,storage,test]
+django<3
 
 
 # xForm and data manipulation
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index 88ac34c68..2fbea0af3 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -14,10 +14,10 @@
 
 aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.14.0
-botocore==1.17.0
+boto3==1.14.9
+botocore==1.17.9
 cachetools==4.1.0
-certifi==2020.4.5.2
+certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
@@ -26,7 +26,7 @@ cryptography==2.9.2
 Django==2.2.13
 django-cacheops==5.0
 django-cleanup==5.0.0
-django-cors-headers==3.3.0
+django-cors-headers==3.4.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-postgrespool2==1.0.1
@@ -42,15 +42,14 @@ flake8==3.8.3
 flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
-google-api-core==1.20.0
-google-auth==1.17.0
+google-api-core==1.21.0
+google-auth==1.18.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.1
+google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.9
-importlib-metadata==1.6.1
 Jinja2==2.11.2
 jmespath==0.10.0
 linecache2==1.0.0
@@ -73,10 +72,10 @@ python-json-logger==0.1.11
 pytz==2020.1
 pyxform==1.1.0
 redis==3.5.3
-requests==2.23.0
-rsa==4.1
+requests==2.24.0
+rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.14.4
+sentry-sdk==0.15.1
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.17
@@ -87,6 +86,5 @@ traceback2==1.4.0
 unicodecsv==0.14.1
 unittest2==1.1.0
 urllib3==1.25.9
-uWSGI==2.0.18
+uWSGI==2.0.19.1
 xlrd==1.2.0
-zipp==3.1.0
diff --git a/aether-producer/Dockerfile b/aether-producer/Dockerfile
index e525a2fc3..87cd204f1 100644
--- a/aether-producer/Dockerfile
+++ b/aether-producer/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Kafka Producer" \
       name="aether-producer" \
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index 86353316a..f55b41408 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -13,7 +13,7 @@
 ################################################################################
 
 attrs==19.3.0
-certifi==2020.4.5.2
+certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 click==7.1.2
@@ -22,33 +22,31 @@ cryptography==2.9.2
 flake8==3.8.3
 flake8-quotes==3.2.0
 Flask==1.1.2
-gevent==20.6.1
+gevent==20.6.2
 greenlet==0.4.16
 idna==2.9
-importlib-metadata==1.6.1
 itsdangerous==1.1.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
-more-itertools==8.3.0
+more-itertools==8.4.0
 packaging==20.4
 pluggy==0.13.1
 psycogreen==1.0.2
 psycopg2-binary==2.8.5
-py==1.8.1
+py==1.8.2
 pycodestyle==2.6.0
 pycparser==2.20
 pyflakes==2.2.0
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
 pytest==5.4.3
-requests==2.23.0
+requests==2.24.0
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.17
 urllib3==1.25.9
-wcwidth==0.2.4
+wcwidth==0.2.5
 Werkzeug==1.0.1
-zipp==3.1.0
 zope.event==4.4
 zope.interface==5.1.0
diff --git a/aether-ui/Dockerfile b/aether-ui/Dockerfile
index d52b9eb04..adbdc1ec2 100644
--- a/aether-ui/Dockerfile
+++ b/aether-ui/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Kernel UI" \
       name="aether-ui" \
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index d16a3a3c9..e4faba27d 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -24,18 +24,18 @@
     "bootstrap": "~4.5.0",
     "html5shiv": "~3.7.0",
     "jquery": "~3.5.0",
-    "moment": "~2.26.0",
+    "moment": "~2.27.0",
     "popper.js": "~1.16.0",
     "react": "~16.13.0",
     "react-clipboard.js": "~2.0.16",
     "react-dom": "~16.13.0",
-    "react-intl": "~4.6.0",
+    "react-intl": "~4.7.0",
     "react-outside-click-handler": "~1.3.0",
     "react-redux": "~7.2.0",
     "react-router-dom": "~5.2.0",
     "redux": "~4.0.0",
     "redux-thunk": "~2.3.0",
-    "uuid": "~8.1.0",
+    "uuid": "~8.2.0",
     "webpack-google-cloud-storage-plugin": "~0.9.0",
     "webpack-s3-plugin": "~1.0.3",
     "whatwg-fetch": "~3.0.0"
@@ -47,12 +47,12 @@
     "@babel/preset-react": "~7.10.0",
     "@hot-loader/react-dom": "~16.13.0",
     "babel-loader": "~8.1.0",
-    "css-loader": "~3.5.0",
-    "eslint": "~7.2.0",
+    "css-loader": "~3.6.0",
+    "eslint": "~7.3.0",
     "enzyme": "~3.11.0",
     "enzyme-adapter-react-16": "~1.15.0",
     "express": "~4.17.0",
-    "jest": "~26.0.0",
+    "jest": "~26.1.0",
     "mini-css-extract-plugin": "~0.9.0",
     "nock": "~12.0.0",
     "node-fetch": "~2.6.0",
diff --git a/aether-ui/conf/pip/primary-requirements.txt b/aether-ui/conf/pip/primary-requirements.txt
index 47e9aa07f..14e46bc2a 100644
--- a/aether-ui/conf/pip/primary-requirements.txt
+++ b/aether-ui/conf/pip/primary-requirements.txt
@@ -20,4 +20,5 @@ aether.sdk[cache,server,webpack,storage,test]
 
 
 # Django specific
+django<3
 django-model-utils
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index 3ee6f004d..a7e831cf3 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -14,10 +14,10 @@
 
 aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.14.0
-botocore==1.17.0
+boto3==1.14.9
+botocore==1.17.9
 cachetools==4.1.0
-certifi==2020.4.5.2
+certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
@@ -26,7 +26,7 @@ cryptography==2.9.2
 Django==2.2.13
 django-cacheops==5.0
 django-cleanup==5.0.0
-django-cors-headers==3.3.0
+django-cors-headers==3.4.0
 django-debug-toolbar==2.2
 django-minio-storage==0.3.7
 django-model-utils==4.0.0
@@ -43,15 +43,14 @@ drf-dynamic-fields==0.3.1
 flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.20.0
-google-auth==1.17.0
+google-api-core==1.21.0
+google-auth==1.18.0
 google-cloud-core==1.3.0
-google-cloud-storage==1.28.1
+google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.9
-importlib-metadata==1.6.1
 Jinja2==2.11.2
 jmespath==0.10.0
 MarkupSafe==1.1.1
@@ -71,15 +70,14 @@ python-dateutil==2.8.1
 python-json-logger==0.1.11
 pytz==2020.1
 redis==3.5.3
-requests==2.23.0
-rsa==4.1
+requests==2.24.0
+rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.14.4
+sentry-sdk==0.15.1
 six==1.15.0
 SQLAlchemy==1.3.17
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1
 urllib3==1.25.9
-uWSGI==2.0.18
-zipp==3.1.0
+uWSGI==2.0.19.1
diff --git a/scripts/deployment/kernel.Dockerfile b/scripts/deployment/kernel.Dockerfile
index 2db08d15b..d7e0caa21 100644
--- a/scripts/deployment/kernel.Dockerfile
+++ b/scripts/deployment/kernel.Dockerfile
@@ -14,7 +14,7 @@ RUN /tmp/setup_revision.sh
 ## using python image to build app
 ################################################################################
 
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Kernel" \
       name="aether-kernel" \
diff --git a/scripts/deployment/odk.Dockerfile b/scripts/deployment/odk.Dockerfile
index 5c67f698b..045d3f21e 100644
--- a/scripts/deployment/odk.Dockerfile
+++ b/scripts/deployment/odk.Dockerfile
@@ -14,7 +14,7 @@ RUN /tmp/setup_revision.sh
 ## using python image to build app
 ################################################################################
 
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether ODK Module" \
       name="aether-odk" \
diff --git a/scripts/deployment/producer.Dockerfile b/scripts/deployment/producer.Dockerfile
index ebddab5b8..54336e119 100644
--- a/scripts/deployment/producer.Dockerfile
+++ b/scripts/deployment/producer.Dockerfile
@@ -14,7 +14,7 @@ RUN /tmp/setup_revision.sh
 ## using python image to build app
 ################################################################################
 
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Kafka Producer" \
       name="aether-producer" \
diff --git a/scripts/deployment/ui.Dockerfile b/scripts/deployment/ui.Dockerfile
index 324d50cb2..339bafeea 100644
--- a/scripts/deployment/ui.Dockerfile
+++ b/scripts/deployment/ui.Dockerfile
@@ -30,7 +30,7 @@ RUN npm install -q && npm run build
 ## using python image to build app
 ################################################################################
 
-FROM python:3.7-slim-buster AS app
+FROM python:3.8-slim-buster AS app
 
 LABEL description="Aether Kernel UI" \
       name="aether-ui" \
diff --git a/test-aether-integration-module/Dockerfile b/test-aether-integration-module/Dockerfile
index 757a7e5dc..eb898aa9f 100644
--- a/test-aether-integration-module/Dockerfile
+++ b/test-aether-integration-module/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 LABEL description="Aether Integration Tests" \
       name="aether-integration-test" \

From f24a49bfdfa8c6e26d8ac18d8fbb49245c516884 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 24 Jun 2020 10:48:58 +0200
Subject: [PATCH 17/29] fix(odk): xml choices (#859)

* fix: (odk) xlm choices (#810)

* fix: (odk) xlm choices to avro schema

* test: (odk) choices from xml

* fix: (odk) exact or default choices schema

* fix: cleaning

* fix: typo

Co-authored-by: Lord-Mallam <lordy002000@gmail.com>
---
 .../aether/kernel/api/tests/test_views.py     |   2 +-
 .../aether/odk/api/tests/test_xform_utils.py  | 144 ++++++++++++++++++
 .../aether/odk/api/xform_utils.py             |  15 +-
 3 files changed, 157 insertions(+), 4 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py
index 49ad21ec6..180dc1499 100644
--- a/aether-kernel/aether/kernel/api/tests/test_views.py
+++ b/aether-kernel/aether/kernel/api/tests/test_views.py
@@ -800,7 +800,7 @@ def test_schema_unique_usage(self):
         response = self.client.post(url, data, content_type='application/json')
         self.assertEqual(response.status_code, 500)
 
-    def test_entity__submit_mutiple__success(self):
+    def test_entity__submit_multiple__success(self):
         response = self.client.post(reverse('entity-list'),
                                     json.dumps([]),
                                     content_type='application/json')
diff --git a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
index bfee64c73..54d349eb3 100644
--- a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
+++ b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
@@ -20,8 +20,10 @@
 
 from . import CustomTestCase
 from ..xform_utils import (
+    __find_by_key_value as find_value,
     __get_all_paths as get_paths,
     __get_avro_primitive_type as get_type,
+    __get_xform_choices as get_choices,
     __get_xform_instance as get_instance,
     __get_xform_itexts as get_texts,
     __get_xform_label as get_label,
@@ -1132,3 +1134,145 @@ def test__parse_xform_to_avro_schema_type_decoration(self):
         project_name = 'TestProject'
         schema = parse_xform_to_avro_schema(xml_definition, project_name)
         self.assertEqual(schema, expected, json.dumps(schema, indent=2))
+
+    def test__find_by_key_value(self):
+        xform_dict = {
+            'h:html': {
+                'h:body': {
+                    'any-tag': {
+                        '@ref': 'humidity',
+                        'item': [
+                            {
+                                'label': 'Dry or low',
+                                'value': 'low'
+                            },
+                            {
+                                'label': 'Normal or medium',
+                                'value': 'med'
+                            },
+                            {
+                                'label': 'Wet or High',
+                                'value': 'high'
+                            }
+                        ],
+                        'label': 'Humidity:'
+                    },
+                    'another-tag': {
+                        '@ref': '/nm/a/b/humidity',
+                        'item': [
+                            {
+                                'label': 'Another Dry or low',
+                                'value': 'another-low'
+                            },
+                            {
+                                'label': 'Another Normal or medium',
+                                'value': 'another-med'
+                            },
+                            {
+                                'label': 'Another Wet or High',
+                                'value': 'another-high'
+                            }
+                        ],
+                        'label': 'Humidity:'
+                    },
+                    'wrong-tag': {
+                        '@ref': '/nm/a/b/humidity-wrong',
+                        'item': [
+                            {
+                                'label': 'Wrong Dry or low',
+                                'value': 'wrong-low'
+                            },
+                            {
+                                'label': 'Wrong Normal or medium',
+                                'value': 'wrong-med'
+                            },
+                            {
+                                'label': 'Wrong Wet or High',
+                                'value': 'wrong-high'
+                            }
+                        ],
+                        'label': 'Humidity:'
+                    }
+                }
+            }
+        }
+        found_nodes = list(find_value(xform_dict, '@ref', '/nm/a/b/humidity', True))
+        self.assertEqual(len(found_nodes), 2)
+
+    def test__get_choices(self):
+        expected = [
+            {
+                'label': 'Dry or low',
+                'value': 'low'
+            },
+            {
+                'label': 'Normal or medium',
+                'value': 'med'
+            },
+            {
+                'label': 'Wet or High',
+                'value': 'high'
+            }
+        ]
+        expected_wrong = [
+            {
+                'label': 'Wrong Dry or low',
+                'value': 'wrong-low'
+            },
+            {
+                'label': 'Wrong Normal or medium',
+                'value': 'wrong-med'
+            },
+            {
+                'label': 'Wrong Wet or High',
+                'value': 'wrong-high'
+            }
+        ]
+        expected_default = [
+            {
+                'label': 'Another Dry or low',
+                'value': 'another-low'
+            },
+            {
+                'label': 'Another Normal or medium',
+                'value': 'another-med'
+            },
+            {
+                'label': 'Another Wet or High',
+                'value': 'another-high'
+            }
+        ]
+
+        xform_dict = {
+            'h:html': {
+                'h:body': {
+                    'another-tag': {
+                        '@ref': 'humidity',
+                        'item': expected_default,
+                        'label': 'Humidity:'
+                    },
+                    'any-tag': {
+                        '@ref': '/a/b/c/humidity',
+                        'item': expected,
+                        'label': 'Humidity:'
+                    },
+                    'wrong-tag': {
+                        '@ref': '/nm/a/b/humidity-wrong',
+                        'item': expected_wrong,
+                        'label': 'Humidity:'
+                    }
+                }
+            }
+        }
+
+        choices = get_choices(xform_dict, '/a/b/c/humidity')
+        self.assertEqual(choices, expected)
+
+        choices = get_choices(xform_dict, '/nm/a/b/humidity-wrong')
+        self.assertEqual(choices, expected_wrong)
+
+        choices = get_choices(xform_dict, '/a/b/c/does-not-exist')
+        self.assertIsNone(choices)
+
+        choices = get_choices(xform_dict, '/unknown/humidity')
+        self.assertEqual(choices, expected_default)
diff --git a/aether-odk-module/aether/odk/api/xform_utils.py b/aether-odk-module/aether/odk/api/xform_utils.py
index d6f7e0985..f7ee5d715 100644
--- a/aether-odk-module/aether/odk/api/xform_utils.py
+++ b/aether-odk-module/aether/odk/api/xform_utils.py
@@ -713,7 +713,12 @@ def __get_xform_instance_skeleton(xml_definition):
 
 
 def __get_xform_choices(xform_dict, xpath, texts={}):
-    select_node = list(__find_by_key_value(xform_dict, '@ref', xpath))[0]
+    found_nodes = list(__find_by_key_value(xform_dict, '@ref', xpath, True))
+    if len(found_nodes) > 1:
+        exact_node = [d for d in found_nodes if d['@ref'] == xpath]
+        select_node = exact_node[0] if exact_node else found_nodes[0]
+    else:
+        select_node = found_nodes[0] if found_nodes else {}
     select_options = __wrap_as_list(select_node.get('item', []))
 
     # limitation: skips selects linked to a datasource with 'itemset'
@@ -951,13 +956,17 @@ def __find_in_dict(dictionary, key):
             yield result
 
 
-def __find_by_key_value(dictionary, key, value):
+def __find_by_key_value(dictionary, key, value, has_options=False):
+    last_node = value.split('/')[-1] if has_options else None
+
     for k, v in dictionary.items():
         if k == key and v == value:
             yield dictionary
+        elif has_options and k == key and v == last_node:
+            yield dictionary
 
         # continue searching in the value keys
-        for result in __iterate_dict(v, __find_by_key_value, key, value):
+        for result in __iterate_dict(v, __find_by_key_value, key, value, has_options):
             yield result
 
 

From 0e873fe4047aaf615203808a654f6aa709a721a7 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 24 Jun 2020 11:58:44 +0200
Subject: [PATCH 18/29] chore: LOGGING_LEVEL and named volumes in docker (#861)

* chore: set common LOGGING_LEVEL

* chore: named volumes in docker settings

* fix: typo
---
 aether-kernel/aether/kernel/api/exporter.py            |  6 +++---
 aether-kernel/aether/kernel/api/models.py              |  2 +-
 aether-kernel/aether/kernel/api/project_artefacts.py   |  2 +-
 .../aether/kernel/api/tests/test_mapping_validation.py |  2 +-
 .../aether/odk/api/collect/authentication.py           |  2 +-
 aether-odk-module/aether/odk/api/collect/views.py      |  2 +-
 aether-odk-module/aether/odk/api/kernel_utils.py       |  2 +-
 .../aether/odk/api/tests/test_xform_utils.py           |  4 ++--
 aether-odk-module/aether/odk/api/xform_utils.py        |  2 +-
 aether-producer/README.md                              | 10 +++++-----
 aether-producer/producer/db.py                         |  2 +-
 aether-producer/producer/kernel.py                     |  4 ++--
 aether-producer/producer/topic.py                      |  4 ++--
 .../assets/apps/pipeline/components/DeleteStatus.jsx   |  2 +-
 .../aether/ui/assets/apps/pipeline/sections/Input.jsx  |  4 ++--
 docker-compose-base.yml                                |  6 +++++-
 docker-compose-test.yml                                |  8 ++++++++
 docker-compose.yml                                     |  4 +++-
 scripts/build_docker_credentials.sh                    |  2 ++
 19 files changed, 43 insertions(+), 27 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/exporter.py b/aether-kernel/aether/kernel/api/exporter.py
index 39533a5de..d25307c97 100644
--- a/aether-kernel/aether/kernel/api/exporter.py
+++ b/aether-kernel/aether/kernel/api/exporter.py
@@ -1069,7 +1069,7 @@ def __filter_headers(paths, group, headers):
 def __order_headers(headers):
     '''
     ISSUE: order the headers so the nested arrays appear together and in order.
-    The sorting algorithm MUST be STABLE (luckly python "sorted" method is).
+    The sorting algorithm MUST be STABLE (luckily python "sorted" method is).
 
     ASSUMPTIONS:
         a) The are no numeric properties outside the nested list items. (AVRO naming restrictions)
@@ -1082,13 +1082,13 @@ def __order_headers(headers):
         ``1`` and ``3`` without ``2``:
             [ ..., "any_path.1.and_more", ..., "any_path.3.and_more",... ]
 
-    IMPLEMENTED SOLUTION: assign a weigth to each list element
+    IMPLEMENTED SOLUTION: assign a weight to each list element
 
         A) it's not a flatten array element
                 then set its position in the array
 
         B) it's a flatten array element
-              then set the same weigth as the first flatten element in the list
+              then set the same weight as the first flatten element in the list
               along with its index (recursively)
               For "any_path.3.and_more" set ("any_path" row, 3) pair
 
diff --git a/aether-kernel/aether/kernel/api/models.py b/aether-kernel/aether/kernel/api/models.py
index 0e8918096..7150c2c6f 100644
--- a/aether-kernel/aether/kernel/api/models.py
+++ b/aether-kernel/aether/kernel/api/models.py
@@ -421,7 +421,7 @@ class Schema(ExportModelOperationsMixin('kernel_schema'), KernelAbstract):
     .. note:: Extends from :class:`aether.kernel.api.models.KernelAbstract`
 
     :ivar text      type:        Schema namespace
-    :ivar JSON      definition:  AVRO schema definiton.
+    :ivar JSON      definition:  AVRO schema definition.
     :ivar text      family:      Schema family.
 
     '''
diff --git a/aether-kernel/aether/kernel/api/project_artefacts.py b/aether-kernel/aether/kernel/api/project_artefacts.py
index f409c7409..7589728ba 100644
--- a/aether-kernel/aether/kernel/api/project_artefacts.py
+++ b/aether-kernel/aether/kernel/api/project_artefacts.py
@@ -274,7 +274,7 @@ def __upsert_instance(model, pk=None, ignore_fields=[], action='upsert', unique_
         # An example would be the AVRO schema generated by the ODK module
         # and the mapping rules generated by the UI app.
         # This is a collaborative app where each part has a main task and all
-        # of them work together to achive a common goal.
+        # of them work together to achieve a common goal.
         try:
             item = model.objects.get(pk=pk)
             is_new = False
diff --git a/aether-kernel/aether/kernel/api/tests/test_mapping_validation.py b/aether-kernel/aether/kernel/api/tests/test_mapping_validation.py
index 61f0d7d4b..e1c31b440 100644
--- a/aether-kernel/aether/kernel/api/tests/test_mapping_validation.py
+++ b/aether-kernel/aether/kernel/api/tests/test_mapping_validation.py
@@ -147,7 +147,7 @@ def test_validate_setter__success__set_array(self):
         result = mapping_validation.validate_setter(valid_schemas, path)
         self.assertEqual(expected, result)
 
-    def test_validate_setter__success__set_array_at_idndex(self):
+    def test_validate_setter__success__set_array_at_index(self):
         path = 'Nested.geom.coordinates[0]'
         expected = mapping_validation.Success(path, [])
         result = mapping_validation.validate_setter(valid_schemas, path)
diff --git a/aether-odk-module/aether/odk/api/collect/authentication.py b/aether-odk-module/aether/odk/api/collect/authentication.py
index ac6ea5525..1e232a621 100644
--- a/aether-odk-module/aether/odk/api/collect/authentication.py
+++ b/aether-odk-module/aether/odk/api/collect/authentication.py
@@ -29,7 +29,7 @@
 
 class CollectAuthentication(BaseAuthentication):
     '''
-    Use Basic or Digest authentication depending on the autorization header.
+    Use Basic or Digest authentication depending on the authorization header.
     '''
 
     def authenticate(self, request):
diff --git a/aether-odk-module/aether/odk/api/collect/views.py b/aether-odk-module/aether/odk/api/collect/views.py
index a015a6680..1a7fa7011 100644
--- a/aether-odk-module/aether/odk/api/collect/views.py
+++ b/aether-odk-module/aether/odk/api/collect/views.py
@@ -146,7 +146,7 @@
 
 def _get_host(request, current_path):
     # ODK Collect needs the full URL to get the resources. They have only the path
-    # like /my/resouce/path/id but not the scheme or the host name,
+    # like /my/resource/path/id but not the scheme or the host name,
     # using the current path we try to figure out the real host to build the
     # linked URLs in the XML templates.
     #
diff --git a/aether-odk-module/aether/odk/api/kernel_utils.py b/aether-odk-module/aether/odk/api/kernel_utils.py
index 4a9c6b840..b415fb8ee 100644
--- a/aether-odk-module/aether/odk/api/kernel_utils.py
+++ b/aether-odk-module/aether/odk/api/kernel_utils.py
@@ -85,7 +85,7 @@ def propagate_kernel_project(project, family=None):
         - one Project,
         - one Mapping,
         - one Schema and
-        - one SchemaDecoratror.
+        - one SchemaDecorator.
     '''
 
     artefacts = {
diff --git a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
index 54d349eb3..097ed67fd 100644
--- a/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
+++ b/aether-odk-module/aether/odk/api/tests/test_xform_utils.py
@@ -1009,7 +1009,7 @@ def test__parse_xform_to_avro_schema_type_decoration(self):
                     </item>
                     <item>
                         <label>Non-residential</label>
-                        <value>non_residentia</value>
+                        <value>non_residential</value>
                     </item>
                     <item>
                         <label>Mixed</label>
@@ -1117,7 +1117,7 @@ def test__parse_xform_to_avro_schema_type_decoration(self):
                         },
                         {
                             'label': 'Non-residential',
-                            'value': 'non_residentia'
+                            'value': 'non_residential'
                         },
                         {
                             'label': 'Mixed',
diff --git a/aether-odk-module/aether/odk/api/xform_utils.py b/aether-odk-module/aether/odk/api/xform_utils.py
index f7ee5d715..315842e83 100644
--- a/aether-odk-module/aether/odk/api/xform_utils.py
+++ b/aether-odk-module/aether/odk/api/xform_utils.py
@@ -773,7 +773,7 @@ def __get_xform_itexts(xform_dict):
             translation = tt
             break
 
-    # convert all text entries in a dict wich key is the text id
+    # convert all text entries in a dict which key is the text id
     itexts = {}
     for text_entry in __wrap_as_list(translation.get('text')):
         for value in __wrap_as_list(text_entry.get('value')):
diff --git a/aether-producer/README.md b/aether-producer/README.md
index 824d008e2..f20ff467b 100644
--- a/aether-producer/README.md
+++ b/aether-producer/README.md
@@ -6,13 +6,13 @@ The Aether Producer (AP) provides a bridge between the RDBMS world of the kernel
 
 #### Method of Operation
 
-The AP interacts directly with the kernel database. It polls to keep state on all registered entity types. Once a type is registered, the AP will continuously query the kernel database for new entities that have been added to that type. 
- - By default all entities of the same type are produced to a Kafka topic bearing the name of that entity type. 
- - By default messages adhereing to the same schema are serialized into Avro objects containing between 1 -> 250 messages to save space.
+The AP interacts directly with the kernel database. It polls to keep state on all registered entity types. Once a type is registered, the AP will continuously query the kernel database for new entities that have been added to that type.
+ - By default all entities of the same type are produced to a Kafka topic bearing the name of that entity type.
+ - By default messages adhering to the same schema are serialized into Avro objects containing between 1 -> 250 messages to save space.
 
 #### Persistence
 
-In kernel, all entities contain a modified datetime field which the AP uses to determine which entities have already been sent to Kafka. The AP refers to this value as an offset, and for each entity type, it maintains a record of the lastest offset that has been accepted by Kafka. On shutdown or failure, the AP will resume polling for at `modified > last_offset`.
+In kernel, all entities contain a modified datetime field which the AP uses to determine which entities have already been sent to Kafka. The AP refers to this value as an offset, and for each entity type, it maintains a record of the latest offset that has been accepted by Kafka. On shutdown or failure, the AP will resume polling for at `modified > last_offset`.
 
 #### Control
 
@@ -23,4 +23,4 @@ While normally the producer can be left to run on its own, sometimes for testing
  - `/resume?topic={topic}` : Resume production on a paused topic
  - `/rebuild?topic={topic}` : Allow a user to force the producer to delete a topic. Requires a subsequent call to `/resume` to then recreate & repopulate the topic. One cannot pause or resume a topic that's in the process of being rebuilt. `/status` for that topic will indicate it's rebuild status.
 
-To allow for this to operate securely, all endpoints besides /healthcheck require authentication via basic auth. There is only currently one user and the credentials are set through environment.
\ No newline at end of file
+To allow for this to operate securely, all endpoints besides /healthcheck require authentication via basic auth. There is only currently one user and the credentials are set through environment.
diff --git a/aether-producer/producer/db.py b/aether-producer/producer/db.py
index a1929c43b..70d7aca81 100644
--- a/aether-producer/producer/db.py
+++ b/aether-producer/producer/db.py
@@ -54,7 +54,7 @@ class PriorityDatabasePool(object):
     # The large number of requests AEP makes was having a bad time with SQLAlchemy's
     # QueuePool implementation, causing a large number of connections in the pool to
     # be occupied, eventually overflowing. Additionally, a normal FIFO queue was
-    # causing imporant operations like offset.set() to sit idle for longer than acceptable.
+    # causing important operations like offset.set() to sit idle for longer than acceptable.
     # The priority queue is implemented to respect both stated priority and insert order.
     # So effectively for each priority level, a FIFO queue is implemented.
 
diff --git a/aether-producer/producer/kernel.py b/aether-producer/producer/kernel.py
index d9595d87c..7ad07b086 100644
--- a/aether-producer/producer/kernel.py
+++ b/aether-producer/producer/kernel.py
@@ -43,8 +43,8 @@ def get_time_window_filter(self, query_time):
         # based on the insert time and now() to provide a buffer.
 
         def fn(row):
-            commited = datetime.strptime(row.get('modified')[:26], _TIME_FORMAT)
-            lag_time = (query_time - commited).total_seconds()
+            committed = datetime.strptime(row.get('modified')[:26], _TIME_FORMAT)
+            lag_time = (query_time - committed).total_seconds()
             if lag_time > _WINDOW_SIZE_SEC:
                 return True
 
diff --git a/aether-producer/producer/topic.py b/aether-producer/producer/topic.py
index 3c726c2f7..eccbd24d0 100644
--- a/aether-producer/producer/topic.py
+++ b/aether-producer/producer/topic.py
@@ -254,7 +254,7 @@ def kafka_callback(self, err=None, msg=None, _=None, **kwargs):
     def update_kafka(self):
         # Main update loop
         # Monitors postgres for changes via TopicManager.updates_available
-        # Consumes updates to the Posgres DB via TopicManager.get_db_updates
+        # Consumes updates to the Postgres DB via TopicManager.get_db_updates
         # Sends new messages to Kafka
         # Registers message callback (ok or fail) to TopicManager.kafka_callback
         # Waits for all messages to be accepted or timeout in TopicManager.wait_for_kafka
@@ -359,7 +359,7 @@ def wait_for_kafka(self, end_offset, timeout=10, iters_per_sec=10, failure_wait_
                 logger.debug(f'All changes saved ok in topic {self.name}.')
                 break
 
-            # Remove successful and errored changes
+            # Remove successful and failed changes
             for k in self.failed_changes:
                 try:
                     del self.change_set[k]
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
index 617e20b38..f11ca7993 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/components/DeleteStatus.jsx
@@ -166,7 +166,7 @@ const DeleteStatus = ({
             <label className='form-label mt-4'>
               <span className='badge badge-b'>{deleteStatus.submissions}</span>
               <FormattedMessage
-                id='delete.modal.sumbissions.status'
+                id='delete.modal.submissions.status'
                 defaultMessage='Submissions deleted'
               />
             </label>
diff --git a/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx b/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
index 8b6c4c7dc..7b510bdee 100644
--- a/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
+++ b/aether-ui/aether/ui/assets/apps/pipeline/sections/Input.jsx
@@ -36,7 +36,7 @@ const MESSAGES = defineMessages({
     defaultMessage: 'Avro Schema',
     id: 'input.schema.view'
   },
-  avroSchemaPlacehoder: {
+  avroSchemaPlaceholder: {
     defaultMessage: 'Paste an AVRO Schema and Sample Data will be generated for your convenience to use in the pipeline.',
     id: 'input.schema.placeholder'
   },
@@ -177,7 +177,7 @@ const Input = ({ pipeline, highlight, updatePipeline }) => {
     }
   }
 
-  const placeholder = view === DATA_VIEW ? MESSAGES.inputDataPlaceholder : MESSAGES.avroSchemaPlacehoder
+  const placeholder = view === DATA_VIEW ? MESSAGES.inputDataPlaceholder : MESSAGES.avroSchemaPlaceholder
   const submitLabel = view === DATA_VIEW ? MESSAGES.inputDataSubmitButton : MESSAGES.avroSchemaSubmitButton
 
   const setValue = view === DATA_VIEW ? setInputStr : setSchemaStr
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index 3b25c3363..970a6a54c 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -9,7 +9,7 @@
 #  * Aether Kafka Producer
 #
 # These container will be extended in the other DC files with dependencies and networks.
-# Volumes and environment variables can be overriden in those files too.
+# Volumes and environment variables can be overridden in those files too.
 # See more in: https://docs.docker.com/compose/extends/
 # ------------------------------------------------------------------------------
 
@@ -93,6 +93,7 @@ services:
       CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
       DJANGO_SECRET_KEY: ${KERNEL_DJANGO_SECRET_KEY}
       LOGGING_FORMATTER: verbose
+      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
       PROFILING_ENABLED: "true"
 
@@ -171,6 +172,7 @@ services:
       CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
       DJANGO_SECRET_KEY: ${ODK_DJANGO_SECRET_KEY}
       LOGGING_FORMATTER: verbose
+      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
       PROFILING_ENABLED: "true"
 
@@ -243,6 +245,7 @@ services:
       CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
       DJANGO_SECRET_KEY: ${UI_DJANGO_SECRET_KEY}
       LOGGING_FORMATTER: verbose
+      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
       PROFILING_ENABLED: "true"
 
@@ -333,6 +336,7 @@ services:
     stdin_open: true
     environment:
       PYTHONUNBUFFERED: 1
+      LOG_LEVEL: ${LOGGING_LEVEL:-ERROR}
 
       PRODUCER_ADMIN_USER: ${PRODUCER_ADMIN_USER}
       PRODUCER_ADMIN_PW: ${PRODUCER_ADMIN_PW}
diff --git a/docker-compose-test.yml b/docker-compose-test.yml
index 3594a4b19..0d15de700 100644
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@@ -12,6 +12,10 @@
 
 version: "2.4"
 
+volumes:
+  db_data_test: {}
+  redis_data_test: {}
+
 services:
 
   # ---------------------------------
@@ -22,11 +26,15 @@ services:
     extends:
       file: ./docker-compose-base.yml
       service: postgres-base
+    volumes:
+      - db_data_test:/var/lib/postgresql/data
 
   redis-test:
     extends:
       file: ./docker-compose-base.yml
       service: redis-base
+    volumes:
+      - redis_data_test:/var/lib/redis/data
 
 
   # ---------------------------------
diff --git a/docker-compose.yml b/docker-compose.yml
index 2a03e23d6..75b820af0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,7 +21,7 @@ volumes:
   database_data:
     external:
       name: ${DB_VOLUME}
-
+  redis_data: {}
 
 services:
 
@@ -42,6 +42,8 @@ services:
     extends:
       file: ./docker-compose-base.yml
       service: redis-base
+    volumes:
+      - redis_data:/var/lib/redis/data
     networks:
       - internal
 
diff --git a/scripts/build_docker_credentials.sh b/scripts/build_docker_credentials.sh
index 817bdb1d2..cc5bd3fee 100755
--- a/scripts/build_docker_credentials.sh
+++ b/scripts/build_docker_credentials.sh
@@ -155,6 +155,8 @@ CLIENT_REALM=test
 # ------------------------------------------------------------------
 # Other
 # ==================================================================
+LOGGING_LEVEL=INFO
+
 # set to 1 to disable parallel execution
 TEST_PARALLEL=
 TEST_WORKERS=5

From f457fd2c5ff13c8cebf19f6cb33355e066884c6e Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Thu, 25 Jun 2020 19:12:15 +0200
Subject: [PATCH 19/29] test: set dynamic priority to task execution (#862)

* test: set  dynamic priority to task execution

* fix: clean scripts

* fix: improve admin section

* fix: more cleaing

* fix: improve performance

* fix: tweak django silk
---
 aether-client-library/setup.py               |  2 -
 aether-kernel/aether/kernel/admin.py         | 32 +++++++--
 aether-kernel/aether/kernel/settings.py      | 19 ++++++
 aether-odk-module/aether/odk/admin.py        | 35 ++++++----
 aether-producer/entrypoint.sh                |  2 +-
 aether-ui/aether/ui/admin.py                 | 13 ++++
 docker-compose-base.yml                      |  8 ++-
 docker-compose-locust.yml                    |  7 ++
 scripts/build_all_containers.sh              |  3 +-
 scripts/build_client_and_distribute.sh       |  3 +-
 scripts/build_docker_credentials.sh          |  6 ++
 scripts/clean_all.sh                         |  3 +
 scripts/kill_all.sh                          |  3 +-
 scripts/test_deployment.sh                   |  1 -
 test-aether-integration-module/entrypoint.sh | 11 ++-
 test-aether-integration-module/setup.py      |  3 -
 tests/performance/kernel_taskset.py          | 72 ++++++++++++++------
 tests/performance/settings.py                |  5 ++
 18 files changed, 169 insertions(+), 59 deletions(-)

diff --git a/aether-client-library/setup.py b/aether-client-library/setup.py
index 8357b1a56..c6615ff12 100644
--- a/aether-client-library/setup.py
+++ b/aether-client-library/setup.py
@@ -43,8 +43,6 @@ def read(f):
         'requests[security]',
         'requests_oauthlib'
     ],
-    setup_requires=['pytest-runner'],
-    tests_require=['pytest'],
 
     packages=find_packages(),
     include_package_data=True,
diff --git a/aether-kernel/aether/kernel/admin.py b/aether-kernel/aether/kernel/admin.py
index c2449e99b..a5bbe4bcb 100644
--- a/aether-kernel/aether/kernel/admin.py
+++ b/aether-kernel/aether/kernel/admin.py
@@ -16,54 +16,73 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from django.conf import settings
 from django.contrib import admin
 
 from .api import models, forms
 
+if settings.MULTITENANCY:  # pragma: no cover
+    PROJECT_LIST_FILTER = ('mt__realm',)
+    CHILD_LIST_FILTER = ('project__mt__realm',)
+    ATTACH_LIST_FILTER = ('submission__project__mt__realm',)
+else:  # pragma: no cover
+    PROJECT_LIST_FILTER = []
+    CHILD_LIST_FILTER = []
+    ATTACH_LIST_FILTER = []
+
 
 class BaseAdmin(admin.ModelAdmin):
+    date_hierarchy = 'modified'
     empty_value_display = '---'
     list_per_page = 25
-    readonly_fields = ('id',)
+    readonly_fields = ('id', 'created', 'modified',)
     show_full_result_count = True
 
 
 class ProjectAdmin(BaseAdmin):
-    list_display = ('id', 'name', 'revision',)
+    list_display = ('id', 'name',)
+    list_filter = ('active',) + PROJECT_LIST_FILTER
 
 
 class MappingSetAdmin(BaseAdmin):
     form = forms.MappingSetForm
     list_display = ('id', 'name', 'revision', 'project',)
+    list_filter = CHILD_LIST_FILTER
 
 
 class MappingAdmin(BaseAdmin):
     form = forms.MappingForm
     list_display = ('id', 'name', 'revision', 'mappingset',)
+    list_filter = ('is_active', 'is_read_only',) + CHILD_LIST_FILTER
 
 
 class SubmissionAdmin(BaseAdmin):
     form = forms.SubmissionForm
-    list_display = ('id', 'revision', 'mappingset',)
+    list_display = ('id', 'mappingset', 'is_extracted',)
+    list_filter = ('is_extracted',) + CHILD_LIST_FILTER
 
 
 class AttachmentAdmin(BaseAdmin):
     list_display = ('id', 'name', 'md5sum', 'submission',)
     readonly_fields = ('id', 'md5sum',)
+    list_filter = ATTACH_LIST_FILTER
 
 
 class SchemaAdmin(BaseAdmin):
     form = forms.SchemaForm
-    list_display = ('id', 'name', 'revision',)
+    list_display = ('id', 'name', 'family',)
 
 
 class SchemaDecoratorAdmin(BaseAdmin):
-    list_display = ('id', 'name', 'revision', 'project', 'schema',)
+    list_display = ('id', 'name', 'project', 'schema',)
+    list_filter = CHILD_LIST_FILTER
 
 
 class EntityAdmin(BaseAdmin):
+    date_hierarchy = 'created'
     form = forms.EntityForm
-    list_display = ('id', 'revision', 'status', 'submission', 'mapping',)
+    list_display = ('id', 'status', 'submission', 'mapping',)
+    list_filter = ('status',) + CHILD_LIST_FILTER
 
 
 class ExportTaskAdmin(BaseAdmin):
@@ -72,6 +91,7 @@ class ExportTaskAdmin(BaseAdmin):
         'status_records', 'status_attachments',
     )
     readonly_fields = ('id', 'name',)
+    list_filter = CHILD_LIST_FILTER
 
 
 admin.site.register(models.Project, ProjectAdmin)
diff --git a/aether-kernel/aether/kernel/settings.py b/aether-kernel/aether/kernel/settings.py
index c05c34141..c71413335 100644
--- a/aether-kernel/aether/kernel/settings.py
+++ b/aether-kernel/aether/kernel/settings.py
@@ -23,6 +23,7 @@
 
 from aether.sdk.conf.settings_aether import *  # noqa
 from aether.sdk.conf.settings_aether import (
+    DJANGO_USE_CACHE,
     INSTALLED_APPS,
     MIGRATION_MODULES,
     REST_FRAMEWORK,
@@ -107,3 +108,21 @@
         'drf_yasg.inspectors.StringDefaultFieldInspector',
     ],
 }
+
+# To improve performance
+if DJANGO_USE_CACHE:
+    from aether.sdk.conf.settings_aether import CACHEOPS
+
+    _CACHED_MODULES = [
+        'kernel.attachment',
+        'kernel.entity',
+        'kernel.mapping',
+        'kernel.mappingset',
+        'kernel.project',
+        'kernel.schema',
+        'kernel.schemadecorator',
+        'kernel.submission',
+    ]
+
+    for k in _CACHED_MODULES:
+        CACHEOPS[k] = {'ops': ('fetch', 'get', 'exists')}
diff --git a/aether-odk-module/aether/odk/admin.py b/aether-odk-module/aether/odk/admin.py
index d4d37adb5..cdd7b07d0 100644
--- a/aether-odk-module/aether/odk/admin.py
+++ b/aether-odk-module/aether/odk/admin.py
@@ -16,6 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from django.conf import settings
 from django.contrib import admin, messages
 from django.utils.translation import gettext as _
 
@@ -27,6 +28,15 @@
     propagate_kernel_project,
 )
 
+if settings.MULTITENANCY:  # pragma: no cover
+    PROJECT_LIST_FILTER = ('mt__realm',)
+    XFORM_LIST_FILTER = ('project__mt__realm',)
+    MEDIAFILE_LIST_FILTER = ('xform__project__mt__realm',)
+else:  # pragma: no cover
+    PROJECT_LIST_FILTER = []
+    XFORM_LIST_FILTER = []
+    MEDIAFILE_LIST_FILTER = []
+
 
 class BaseAdmin(admin.ModelAdmin):
 
@@ -53,10 +63,8 @@ def propagate(self, request, queryset):
 
     actions = ['propagate']
     form = ProjectForm
-    list_display = (
-        'project_id',
-        'name',
-    )
+    list_display = ('project_id', 'name', 'active',)
+    list_filter = ('active',) + PROJECT_LIST_FILTER
     search_fields = ('name',)
     ordering = list_display
 
@@ -99,11 +107,16 @@ def propagate(self, request, queryset):
         'title',
         'form_id',
         'description',
-        'created_at',
+        'modified_at',
         'version',
+        'active',
+    )
+    list_filter = ('active',) + XFORM_LIST_FILTER
+    date_hierarchy = 'modified_at'
+    readonly_fields = (
+        'title', 'form_id', 'version', 'md5sum',
+        'avro_schema', 'avro_schema_prettified',
     )
-    date_hierarchy = 'created_at'
-    readonly_fields = ('title', 'form_id', 'version', 'md5sum', 'avro_schema', 'avro_schema_prettified',)
     search_fields = ('project', 'title', 'form_id',)
     ordering = list_display
 
@@ -129,12 +142,8 @@ def propagate(self, request, queryset):
 
 class MediaFileAdmin(BaseAdmin):
 
-    list_display = (
-        'xform',
-        'name',
-        'md5sum',
-        'media_file',
-    )
+    list_display = ('xform', 'name', 'md5sum', 'media_file',)
+    list_filter = MEDIAFILE_LIST_FILTER
     readonly_fields = ('md5sum',)
     search_fields = ('xform', 'name',)
     ordering = list_display
diff --git a/aether-producer/entrypoint.sh b/aether-producer/entrypoint.sh
index 330b5d8f4..8451c1677 100755
--- a/aether-producer/entrypoint.sh
+++ b/aether-producer/entrypoint.sh
@@ -45,7 +45,7 @@ function test_flake8 {
 
 function after_test {
     cat /code/conf/extras/good_job.txt
-    rm -R .pytest_cache
+    rm -rf .pytest_cache
     rm -rf tests/__pycache__
 }
 
diff --git a/aether-ui/aether/ui/admin.py b/aether-ui/aether/ui/admin.py
index d55734abb..ab4d62168 100644
--- a/aether-ui/aether/ui/admin.py
+++ b/aether-ui/aether/ui/admin.py
@@ -17,12 +17,22 @@
 # under the License.
 
 from django import forms
+from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.postgres.forms.jsonb import JSONField
 from django.utils.translation import gettext as _
 
 from .api import models, utils
 
+if settings.MULTITENANCY:  # pragma: no cover
+    PROJECT_LIST_FILTER = ('mt__realm',)
+    PIPELINE_LIST_FILTER = ('project__mt__realm',)
+    CONTRACT_LIST_FILTER = ('pipeline__project__mt__realm',)
+else:  # pragma: no cover
+    PROJECT_LIST_FILTER = []
+    PIPELINE_LIST_FILTER = []
+    CONTRACT_LIST_FILTER = []
+
 
 class PipelineForm(forms.ModelForm):
 
@@ -64,6 +74,7 @@ def publish(self, request, queryset):
     actions = ['publish']
 
     list_display = ('name', 'project_id', 'is_default',)
+    list_filter = ('active',) + PROJECT_LIST_FILTER
     search_fields = list_display
     ordering = list_display
 
@@ -89,6 +100,7 @@ def publish(self, request, queryset):
     form = PipelineForm
 
     list_display = ('name', 'project', 'mappingset',)
+    list_filter = PIPELINE_LIST_FILTER
     search_fields = list_display
     ordering = list_display
 
@@ -117,6 +129,7 @@ def publish(self, request, queryset):
         'name', 'pipeline', 'published_on', 'mapping',
         'is_active', 'is_read_only',
     )
+    list_filter = ('is_active', 'is_read_only',) + CONTRACT_LIST_FILTER
     search_fields = ('name',)
     ordering = list_display
 
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index 970a6a54c..be124378c 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -96,6 +96,10 @@ services:
       LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
       PROFILING_ENABLED: "true"
+      SILKY_PYTHON_PROFILER: "true"
+      SILKY_MAX_REQUEST_BODY_SIZE: 128
+      SILKY_MAX_RESPONSE_BODY_SIZE: 128
+      SILKY_INTERCEPT_PERCENT: 1
 
       KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
       KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
@@ -174,7 +178,7 @@ services:
       LOGGING_FORMATTER: verbose
       LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
-      PROFILING_ENABLED: "true"
+      PROFILING_ENABLED: null
 
       KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
       KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
@@ -247,7 +251,7 @@ services:
       LOGGING_FORMATTER: verbose
       LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
       HTML_SELECT_CUTOFF: 10
-      PROFILING_ENABLED: "true"
+      PROFILING_ENABLED: null
 
       KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
       KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
diff --git a/docker-compose-locust.yml b/docker-compose-locust.yml
index 04f76a240..c07b0d08c 100644
--- a/docker-compose-locust.yml
+++ b/docker-compose-locust.yml
@@ -28,6 +28,13 @@ services:
       BASE_HOST: http://${NETWORK_DOMAIN}
       AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
       AETHER_KERNEL_URL: http://${NETWORK_DOMAIN}/kernel
+
+      # Task priorities
+      CREATE_PROJECT_PRIORITY: ${TEST_CREATE_PROJECT:-2}
+      CREATE_SUBMISSION_PRIORITY: ${TEST_CREATE_SUBMISSION:-100}
+      HEALTH_CHECK_PRIORITY: ${TEST_HEALTH_CHECK:-2}
+      VIEW_PROJECTS_PRIORITY: ${TEST_VIEW_PROJECTS:-5}
+
     volumes: &locust_volumes
       - ./tests/performance:/mnt/locust
     ports:
diff --git a/scripts/build_all_containers.sh b/scripts/build_all_containers.sh
index 3bb1ebb27..1782e6afc 100755
--- a/scripts/build_all_containers.sh
+++ b/scripts/build_all_containers.sh
@@ -31,7 +31,6 @@ create_docker_assets
 build_client
 build_ui_assets
 
-for container in "${containers[@]}"
-do
+for container in "${containers[@]}"; do
     build_container $container
 done
diff --git a/scripts/build_client_and_distribute.sh b/scripts/build_client_and_distribute.sh
index 8c80ec1ae..3ad4fef7e 100755
--- a/scripts/build_client_and_distribute.sh
+++ b/scripts/build_client_and_distribute.sh
@@ -40,8 +40,7 @@ PCK_FILE=aether.client-${APP_VERSION}-py2.py3-none-any.whl
 
 # distribute within the containers
 FOLDERS=( test-aether-integration-module )
-for FOLDER in "${FOLDERS[@]}"
-do
+for FOLDER in "${FOLDERS[@]}"; do
     DEST=./${FOLDER}/conf/pip/dependencies/
     mkdir -p ${DEST}
 
diff --git a/scripts/build_docker_credentials.sh b/scripts/build_docker_credentials.sh
index cc5bd3fee..d1671b93b 100755
--- a/scripts/build_docker_credentials.sh
+++ b/scripts/build_docker_credentials.sh
@@ -161,6 +161,12 @@ LOGGING_LEVEL=INFO
 TEST_PARALLEL=
 TEST_WORKERS=5
 
+# task priorities (higher value higher priority)
+TEST_CREATE_PROJECT=1
+TEST_CREATE_SUBMISSION=100
+TEST_HEALTH_CHECK=2
+TEST_VIEW_PROJECTS=5
+
 # to speed up development changes in the SDK library
 # https://github.com/eHealthAfrica/aether-django-sdk-library
 SDK_PATH=../aether-django-sdk-library
diff --git a/scripts/clean_all.sh b/scripts/clean_all.sh
index df6ff2836..886bb466a 100755
--- a/scripts/clean_all.sh
+++ b/scripts/clean_all.sh
@@ -105,3 +105,6 @@ fi
 if [[ $env = "yes" ]]; then
     rm -f .env
 fi
+
+# clean docker cache
+docker image prune -f
diff --git a/scripts/kill_all.sh b/scripts/kill_all.sh
index fb53041a5..3dbddf310 100755
--- a/scripts/kill_all.sh
+++ b/scripts/kill_all.sh
@@ -20,8 +20,7 @@
 #
 set -Eeuo pipefail
 
-for dc_file in $(find docker-compose*.yml */docker-compose*.yml 2> /dev/null)
-do
+for dc_file in $(find docker-compose*.yml */docker-compose*.yml 2> /dev/null); do
     docker-compose -f $dc_file kill    2> /dev/null || true
     docker-compose -f $dc_file down -v 2> /dev/null || true
 done
diff --git a/scripts/test_deployment.sh b/scripts/test_deployment.sh
index 1f552bd86..1c5f637fe 100755
--- a/scripts/test_deployment.sh
+++ b/scripts/test_deployment.sh
@@ -26,7 +26,6 @@ IMAGE_PREFIX="test-deployment-aether"
 
 for APP in "${DEPLOY_APPS[@]}"; do
     docker build \
-        --force-rm \
         --tag ${IMAGE_PREFIX}-${APP} \
         --file ./scripts/deployment/${APP}.Dockerfile \
         .
diff --git a/test-aether-integration-module/entrypoint.sh b/test-aether-integration-module/entrypoint.sh
index 5b8c1b5af..f2ff0e3b5 100755
--- a/test-aether-integration-module/entrypoint.sh
+++ b/test-aether-integration-module/entrypoint.sh
@@ -32,18 +32,23 @@ function show_help {
     """
 }
 
+function clean_py {
+    rm -rf ./*.egg*
+    rm -rf .pytest_cache
+}
+
 function test_flake8 {
     flake8
 }
 
 function test_python {
-    # Python3 Tests
+    clean_py
+
     export PYTHONDONTWRITEBYTECODE=1
     python3 setup.py -q test "${@:1}"
 
     cat /code/conf/extras/good_job.txt
-    rm -R ./*.egg*
-    rm -R .pytest_cache
+    clean_py
 }
 
 
diff --git a/test-aether-integration-module/setup.py b/test-aether-integration-module/setup.py
index 5dca49182..3af59be8c 100644
--- a/test-aether-integration-module/setup.py
+++ b/test-aether-integration-module/setup.py
@@ -34,7 +34,4 @@ def read(f):
     author='eHealth Africa',
     author_email='aether@ehealthafrica.org',
     license='Apache2 License',
-
-    setup_requires=['pytest-runner'],
-    tests_require=['pytest'],
 )
diff --git a/tests/performance/kernel_taskset.py b/tests/performance/kernel_taskset.py
index 058b448a7..3d0d94e11 100644
--- a/tests/performance/kernel_taskset.py
+++ b/tests/performance/kernel_taskset.py
@@ -21,38 +21,36 @@
 
 from locust import TaskSet, task
 
-from settings import AETHER_KERNEL_URL, AETHER_AUTH_HEADER
+from settings import (
+    AETHER_AUTH_HEADER,
+    AETHER_KERNEL_URL,
 
+    CREATE_PROJECT_PRIORITY,
+    CREATE_SUBMISSION_PRIORITY,
+    HEALTH_CHECK_PRIORITY,
+    VIEW_PROJECTS_PRIORITY,
+)
 
-class KernelTaskSet(TaskSet):
-
-    def on_start(self):
-        response = self.client.get(
-            name='/',
-            headers=AETHER_AUTH_HEADER,
-            url=f'{AETHER_KERNEL_URL}/',
-        )
 
-        # create initial project
-        self.create_avro_schemas()
+class KernelTaskSet(TaskSet):
 
-    @task(1)
-    def health_page(self):
-        response = self.client.get(
+    ###################################################
+    # HELPERS
+    ###################################################
+    def health_check(self):
+        self.client.get(
             name='/health',
             url=f'{AETHER_KERNEL_URL}/health',
         )
 
-    @task(5)
     def view_projects(self):
-        response = self.client.get(
+        self.client.get(
             name='/projects',
             headers=AETHER_AUTH_HEADER,
             url=f'{AETHER_KERNEL_URL}/projects.json',
         )
 
-    @task(2)
-    def create_avro_schemas(self):
+    def create_avro_schema(self):
         project_id = str(uuid.uuid4())
         avro_schema = {
             'name': f'simple-{project_id}',
@@ -69,7 +67,7 @@ def create_avro_schemas(self):
             ],
         }
 
-        response = self.client.request(
+        self.client.request(
             name='/projects/avro-schemas',
             headers=AETHER_AUTH_HEADER,
             method='PATCH',
@@ -80,7 +78,6 @@ def create_avro_schemas(self):
             },
         )
 
-    @task(15)
     def create_submission(self):
         # get list of mapping set ids
         response = self.client.get(
@@ -88,7 +85,6 @@ def create_submission(self):
             name='/mappingsets',
             headers=AETHER_AUTH_HEADER,
         )
-        response.raise_for_status()
         data = response.json()
         if data['count'] == 0:
             return
@@ -105,7 +101,7 @@ def create_submission(self):
             'name': f'Name {submission_id}',
         }
 
-        response = self.client.request(
+        self.client.request(
             name='/submissions',
             headers=AETHER_AUTH_HEADER,
             method='POST',
@@ -116,3 +112,35 @@ def create_submission(self):
                 'payload': submission_payload,
             },
         )
+
+    ###################################################
+    # ON START
+    ###################################################
+    def on_start(self):
+        self.client.get(
+            name='/',
+            headers=AETHER_AUTH_HEADER,
+            url=f'{AETHER_KERNEL_URL}/',
+        )
+
+        # create initial project
+        self.create_avro_schema()
+
+    ###################################################
+    # TASKS
+    ###################################################
+    @task(HEALTH_CHECK_PRIORITY)
+    def task_health_check(self):
+        self.health_check()
+
+    @task(VIEW_PROJECTS_PRIORITY)
+    def task_view_projects(self):
+        self.view_projects()
+
+    @task(CREATE_PROJECT_PRIORITY)
+    def task_create_avro_schema(self):
+        self.create_avro_schema()
+
+    @task(CREATE_SUBMISSION_PRIORITY)
+    def task_create_submission(self):
+        self.create_submission()
diff --git a/tests/performance/settings.py b/tests/performance/settings.py
index c7b83d971..4e51c6005 100644
--- a/tests/performance/settings.py
+++ b/tests/performance/settings.py
@@ -23,3 +23,8 @@
 AETHER_KERNEL_URL = os.environ['AETHER_KERNEL_URL']
 AETHER_KERNEL_TOKEN = os.environ['AETHER_KERNEL_TOKEN']
 AETHER_AUTH_HEADER = {'Authorization': f'Token {AETHER_KERNEL_TOKEN}'}
+
+CREATE_PROJECT_PRIORITY = int(os.environ.get('CREATE_PROJECT_PRIORITY', 1))
+CREATE_SUBMISSION_PRIORITY = int(os.environ.get('CREATE_SUBMISSION_PRIORITY', 100))
+HEALTH_CHECK_PRIORITY = int(os.environ.get('HEALTH_CHECK_PRIORITY', 2))
+VIEW_PROJECTS_PRIORITY = int(os.environ.get('VIEW_PROJECTS_PRIORITY', 5))

From 50c08b424e58091223f631a72911cd7e005a0f13 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Sat, 27 Jun 2020 17:23:54 +0200
Subject: [PATCH 20/29] docs: typo (#866)

* docs: typo

* fix: tweak client setup
---
 .travis.yml                                     | 2 +-
 CONTRIBUTING.md                                 | 4 ++--
 aether-client-library/README.md                 | 2 +-
 aether-client-library/aether/__init__.py        | 5 +----
 aether-client-library/setup.py                  | 1 -
 docker-compose-base.yml                         | 2 +-
 docker-compose-connect.yml                      | 2 +-
 docker-compose-test.yml                         | 2 +-
 test-aether-integration-module/test/consumer.py | 2 +-
 9 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fd79936ab..812606cd4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,7 +31,7 @@ jobs:
       stage: test
       env: 'TEST_MODE=ui'
 
-    - name: "Integration tests (Kernel with Kakfa/Zookeeper and Producer)"
+    - name: "Integration tests (Kernel with Kafka/Zookeeper and Producer)"
       stage: test
       env: 'TEST_MODE=integration'
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 64189f2fb..803fb8310 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,7 +6,7 @@ First off, thank you for considering contributing to Aether. It's people like yo
 
 ## How you can help
 
-Aether is an open source project and we love to receive contributions from our community. There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Aether itself. 
+Aether is an open source project and we love to receive contributions from our community. There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Aether itself.
 
 If you decide that you want to help out with development, you should start by getting Aether installed locally. Follow the [guide](https://aether.ehealthafrica.org/documentation/try/index.html) on the Aether website (and of course, if you have any problems following that guide, or you think it could be improved in any way, feel free to open and issue!). Once you’ve got Aether running and you’ve got a feel for the fundamental concepts and architecture, then you can dive right in and start writing code.
 
@@ -28,7 +28,7 @@ If the answer to either of those two questions are "yes", then you're probably d
 
 When you file an issue, please ensure that you have answered every question in the issue template.
 
-## Getting started with Aether developement
+## Getting started with Aether development
 
 ### Fork and clone the repository
 You will need to fork the main Aether repository and clone it to your local machine. See [github help page](https://help.github.com/articles/fork-a-repo) for help.
diff --git a/aether-client-library/README.md b/aether-client-library/README.md
index 5222ff33f..9dd3b15a6 100644
--- a/aether-client-library/README.md
+++ b/aether-client-library/README.md
@@ -2,4 +2,4 @@
 
 This is the official Python Client for the Aether Kernel.
 
-For usage patterns see `./aether/tests/test_client.py`
+For usage patterns see `./aether/client/test/test.py`
diff --git a/aether-client-library/aether/__init__.py b/aether-client-library/aether/__init__.py
index 41c29ec00..1b53c2af7 100644
--- a/aether-client-library/aether/__init__.py
+++ b/aether-client-library/aether/__init__.py
@@ -16,7 +16,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-try:
-    __import__('pkg_resources').declare_namespace(__name__)
-except ImportError:
-    __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/aether-client-library/setup.py b/aether-client-library/setup.py
index c6615ff12..f0e6c4336 100644
--- a/aether-client-library/setup.py
+++ b/aether-client-library/setup.py
@@ -46,5 +46,4 @@ def read(f):
 
     packages=find_packages(),
     include_package_data=True,
-    namespace_packages=['aether'],
 )
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index be124378c..13ad3a756 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -5,7 +5,7 @@
 #  * Aether Kernel
 #  * ODK Module
 #  * Aether UI & UI Assets
-#  * Zookeper & Kafka
+#  * Zookeeper & Kafka
 #  * Aether Kafka Producer
 #
 # These container will be extended in the other DC files with dependencies and networks.
diff --git a/docker-compose-connect.yml b/docker-compose-connect.yml
index 5f5b2029f..754cb153e 100644
--- a/docker-compose-connect.yml
+++ b/docker-compose-connect.yml
@@ -1,7 +1,7 @@
 # ------------------------------------------------------------------------------
 # Config file for these containers:
 #
-#  * Zookeper & Kafka
+#  * Zookeeper & Kafka
 #  * Aether Kafka Producer
 # ------------------------------------------------------------------------------
 
diff --git a/docker-compose-test.yml b/docker-compose-test.yml
index 0d15de700..bd18b0962 100644
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@@ -5,7 +5,7 @@
 #  * Aether Kernel Client
 #  * ODK Module
 #  * Aether UI
-#  * Zookeper & Kafka
+#  * Zookeeper & Kafka
 #  * Aether Kafka Producer
 #  * Aether Integration Tests
 # ------------------------------------------------------------------------------
diff --git a/test-aether-integration-module/test/consumer.py b/test-aether-integration-module/test/consumer.py
index 5af316ab7..56f097281 100644
--- a/test-aether-integration-module/test/consumer.py
+++ b/test-aether-integration-module/test/consumer.py
@@ -68,7 +68,7 @@ def read(consumer, start='LATEST', verbose=False, timeout_ms=5000, max_records=2
 
 def _read_poll_result(new_records, verbose=False):
     flattened = []
-    for parition_key, packages in new_records.items():
+    for packages in new_records.values():
         for package in packages:
             messages = package.get('messages')
             for msg in messages:

From eefed57c71526f74d57d9eb4ac2805c7270f6dae Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 29 Jun 2020 10:18:02 +0200
Subject: [PATCH 21/29] refactor: move producer to aether.producer (#865)

* refactor: move producer to aether.producer

* fix: last tweaks
---
 aether-producer/aether/__init__.py            | 19 +++++++++++++++++++
 .../{ => aether}/producer/__init__.py         | 10 +++++-----
 aether-producer/{ => aether}/producer/db.py   |  2 +-
 .../{ => aether}/producer/kernel.py           |  2 +-
 .../{ => aether}/producer/kernel_api.py       |  4 ++--
 .../{ => aether}/producer/kernel_db.py        |  6 +++---
 .../{ => aether}/producer/settings.json       |  0
 .../{ => aether}/producer/settings.py         |  0
 .../{ => aether}/producer/topic.py            |  4 ++--
 aether-producer/manage.py                     |  2 +-
 aether-producer/setup.py                      |  6 +++---
 aether-producer/tests/__init__.py             |  4 ++--
 aether-producer/tests/test_integration.py     |  4 ++--
 13 files changed, 41 insertions(+), 22 deletions(-)
 create mode 100644 aether-producer/aether/__init__.py
 rename aether-producer/{ => aether}/producer/__init__.py (96%)
 rename aether-producer/{ => aether}/producer/db.py (99%)
 rename aether-producer/{ => aether}/producer/kernel.py (97%)
 rename aether-producer/{ => aether}/producer/kernel_api.py (98%)
 rename aether-producer/{ => aether}/producer/kernel_db.py (97%)
 rename aether-producer/{ => aether}/producer/settings.json (100%)
 rename aether-producer/{ => aether}/producer/settings.py (100%)
 rename aether-producer/{ => aether}/producer/topic.py (99%)

diff --git a/aether-producer/aether/__init__.py b/aether-producer/aether/__init__.py
new file mode 100644
index 000000000..1b53c2af7
--- /dev/null
+++ b/aether-producer/aether/__init__.py
@@ -0,0 +1,19 @@
+# Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/aether-producer/producer/__init__.py b/aether-producer/aether/producer/__init__.py
similarity index 96%
rename from aether-producer/producer/__init__.py
rename to aether-producer/aether/producer/__init__.py
index ee39da68e..c25b1cec8 100644
--- a/aether-producer/producer/__init__.py
+++ b/aether-producer/aether/producer/__init__.py
@@ -27,15 +27,15 @@
 from gevent.pool import Pool
 from gevent.pywsgi import WSGIServer
 
-from producer.db import init as init_offset_db
-from producer.settings import KAFKA_SETTINGS, SETTINGS, LOG_LEVEL, get_logger
-from producer.topic import KafkaStatus, TopicStatus, TopicManager
+from aether.producer.db import init as init_offset_db
+from aether.producer.settings import KAFKA_SETTINGS, SETTINGS, LOG_LEVEL, get_logger
+from aether.producer.topic import KafkaStatus, TopicStatus, TopicManager
 
 # How to access Kernel: API (default) | DB
 if SETTINGS.get('kernel_access_type', 'api').lower() != 'db':
-    from producer.kernel_api import KernelAPIClient as KernelClient
+    from aether.producer.kernel_api import KernelAPIClient as KernelClient
 else:
-    from producer.kernel_db import KernelDBClient as KernelClient
+    from aether.producer.kernel_db import KernelDBClient as KernelClient
 
 
 class ProducerManager(object):
diff --git a/aether-producer/producer/db.py b/aether-producer/aether/producer/db.py
similarity index 99%
rename from aether-producer/producer/db.py
rename to aether-producer/aether/producer/db.py
index 70d7aca81..bcf47e6ba 100644
--- a/aether-producer/producer/db.py
+++ b/aether-producer/aether/producer/db.py
@@ -43,7 +43,7 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.pool import NullPool
 
-from producer.settings import SETTINGS, get_logger
+from aether.producer.settings import SETTINGS, get_logger
 
 Base = declarative_base()
 logger = get_logger('producer-db')
diff --git a/aether-producer/producer/kernel.py b/aether-producer/aether/producer/kernel.py
similarity index 97%
rename from aether-producer/producer/kernel.py
rename to aether-producer/aether/producer/kernel.py
index 7ad07b086..8012211de 100644
--- a/aether-producer/producer/kernel.py
+++ b/aether-producer/aether/producer/kernel.py
@@ -18,7 +18,7 @@
 
 from datetime import datetime
 
-from producer.settings import SETTINGS, get_logger
+from aether.producer.settings import SETTINGS, get_logger
 
 
 logger = get_logger('producer-kernel')
diff --git a/aether-producer/producer/kernel_api.py b/aether-producer/aether/producer/kernel_api.py
similarity index 98%
rename from aether-producer/producer/kernel_api.py
rename to aether-producer/aether/producer/kernel_api.py
index 2f909c6b2..777f927a9 100644
--- a/aether-producer/producer/kernel_api.py
+++ b/aether-producer/aether/producer/kernel_api.py
@@ -21,8 +21,8 @@
 from datetime import datetime
 from gevent import sleep
 
-from producer.settings import SETTINGS
-from producer.kernel import KernelClient, logger
+from aether.producer.settings import SETTINGS
+from aether.producer.kernel import KernelClient, logger
 
 
 _REQUEST_ERROR_RETRIES = int(SETTINGS.get('request_error_retries', 3))
diff --git a/aether-producer/producer/kernel_db.py b/aether-producer/aether/producer/kernel_db.py
similarity index 97%
rename from aether-producer/producer/kernel_db.py
rename to aether-producer/aether/producer/kernel_db.py
index 51adba35f..d70aac69c 100644
--- a/aether-producer/producer/kernel_db.py
+++ b/aether-producer/aether/producer/kernel_db.py
@@ -30,9 +30,9 @@
 from psycopg2 import sql
 from psycopg2.extras import DictCursor
 
-from producer.db import PriorityDatabasePool
-from producer.settings import SETTINGS
-from producer.kernel import KernelClient, logger
+from aether.producer.db import PriorityDatabasePool
+from aether.producer.settings import SETTINGS
+from aether.producer.kernel import KernelClient, logger
 
 
 _SCHEMAS_SQL = '''
diff --git a/aether-producer/producer/settings.json b/aether-producer/aether/producer/settings.json
similarity index 100%
rename from aether-producer/producer/settings.json
rename to aether-producer/aether/producer/settings.json
diff --git a/aether-producer/producer/settings.py b/aether-producer/aether/producer/settings.py
similarity index 100%
rename from aether-producer/producer/settings.py
rename to aether-producer/aether/producer/settings.py
diff --git a/aether-producer/producer/topic.py b/aether-producer/aether/producer/topic.py
similarity index 99%
rename from aether-producer/producer/topic.py
rename to aether-producer/aether/producer/topic.py
index eccbd24d0..ba4a7b4e6 100644
--- a/aether-producer/producer/topic.py
+++ b/aether-producer/aether/producer/topic.py
@@ -34,8 +34,8 @@
 from spavro.io import DatumWriter, DatumReader
 from spavro.io import validate
 
-from producer.db import Offset
-from producer.settings import SETTINGS, KAFKA_SETTINGS, get_logger
+from aether.producer.db import Offset
+from aether.producer.settings import SETTINGS, KAFKA_SETTINGS, get_logger
 
 logger = get_logger('producer-topic')
 
diff --git a/aether-producer/manage.py b/aether-producer/manage.py
index 31009f0ee..4452ace1e 100755
--- a/aether-producer/manage.py
+++ b/aether-producer/manage.py
@@ -29,6 +29,6 @@
         print('PRODUCER_SETTINGS_FILE not set in environment.')
         sys.exit(1)
 
-    from producer import main
+    from aether.producer import main
     main()
     print('Started Producer with path %s' % settings_path)
diff --git a/aether-producer/setup.py b/aether-producer/setup.py
index 9253098fb..c6136b0d7 100644
--- a/aether-producer/setup.py
+++ b/aether-producer/setup.py
@@ -28,12 +28,12 @@ def read(f):
 
 setup(
     version=read('/var/tmp/VERSION').strip(),
-    name='aether_producer',
-    decription='Kafka Producer for Aether',
+    name='aether.producer',
+    description='Kafka Producer for Aether',
+    keywords=['aet', 'aether', 'kafka', 'producer'],
 
     url='https://github.com/eHealthAfrica/aether',
     author='Shawn Sarwar',
     author_email='shawn.sarwar@ehealthafrica.org',
     license='Apache2 License',
-    keywords=['aet', 'aether', 'kafka', 'producer'],
 )
diff --git a/aether-producer/tests/__init__.py b/aether-producer/tests/__init__.py
index c68163be9..a32d143fa 100644
--- a/aether-producer/tests/__init__.py
+++ b/aether-producer/tests/__init__.py
@@ -18,8 +18,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from producer import ProducerManager
-from producer.settings import SETTINGS, get_logger
+from aether.producer import ProducerManager
+from aether.producer.settings import SETTINGS, get_logger
 
 
 class MockAdminInterface(object):
diff --git a/aether-producer/tests/test_integration.py b/aether-producer/tests/test_integration.py
index c3ecbcd8a..2eef67d47 100644
--- a/aether-producer/tests/test_integration.py
+++ b/aether-producer/tests/test_integration.py
@@ -22,8 +22,8 @@
 import uuid
 from gevent import sleep
 
-from producer.settings import SETTINGS
-from producer.db import Offset
+from aether.producer.settings import SETTINGS
+from aether.producer.db import Offset
 
 from .timeout import timeout as Timeout
 from . import MockProducerManager

From 79c4b56484e7c8111b881617406fd36451179f22 Mon Sep 17 00:00:00 2001
From: shawnsarwar <shawn.sarwar@ehealthnigeria.org>
Date: Wed, 1 Jul 2020 10:03:49 +0200
Subject: [PATCH 22/29] fix: limit producer concurrency (#867)

* refactor: update APIs to handle both single schema calls and the same for all on a realm

* refactor: greatly simplified the kafka publication loop with a better callback

* refactor: realm is now the base unit of parallelism rather than topic

* fix: realm finding time too long for tests when there is no realm yet present
---
 .../aether/client/test/__init__.py            |  14 +
 aether-producer/aether/producer/__init__.py   |  92 +--
 aether-producer/aether/producer/kernel.py     |   6 +
 aether-producer/aether/producer/kernel_api.py | 100 ++-
 aether-producer/aether/producer/kernel_db.py  | 169 +++--
 aether-producer/aether/producer/topic.py      | 626 +++++++++---------
 aether-producer/aether/producer/utils.py      |  41 ++
 aether-producer/tests/__init__.py             |   2 +-
 docker-compose-test.yml                       |   1 -
 .../test/__init__.py                          |  40 +-
 .../test/test_integration.py                  |  21 +-
 11 files changed, 640 insertions(+), 472 deletions(-)
 create mode 100644 aether-producer/aether/producer/utils.py

diff --git a/aether-client-library/aether/client/test/__init__.py b/aether-client-library/aether/client/test/__init__.py
index 51fbe0e43..f26c3b636 100644
--- a/aether-client-library/aether/client/test/__init__.py
+++ b/aether-client-library/aether/client/test/__init__.py
@@ -44,6 +44,20 @@ def simple_entity(value_size=10):
     }
 
 
+@pytest.fixture(scope='session')
+def realm_client():
+    def fn(realm):
+        return Client(
+            URL,
+            USER,
+            PW,
+            log_level=LOG_LEVEL,
+            auth_type='basic',
+            realm=realm
+        )
+    return fn
+
+
 @pytest.fixture(scope='session')
 def client():
     return Client(
diff --git a/aether-producer/aether/producer/__init__.py b/aether-producer/aether/producer/__init__.py
index c25b1cec8..1170e1c90 100644
--- a/aether-producer/aether/producer/__init__.py
+++ b/aether-producer/aether/producer/__init__.py
@@ -29,7 +29,7 @@
 
 from aether.producer.db import init as init_offset_db
 from aether.producer.settings import KAFKA_SETTINGS, SETTINGS, LOG_LEVEL, get_logger
-from aether.producer.topic import KafkaStatus, TopicStatus, TopicManager
+from aether.producer.topic import KafkaStatus, TopicStatus, RealmManager
 
 # How to access Kernel: API (default) | DB
 if SETTINGS.get('kernel_access_type', 'api').lower() != 'db':
@@ -42,8 +42,8 @@ class ProducerManager(object):
     # Serves status & healthcheck over HTTP
     # Dispatches Signals
     # Keeps track of schemas
-    # Spawns a TopicManager for each schema type in Kernel
-    # TopicManager registers own eventloop greenlet (update_kafka) with ProducerManager
+    # Spawns a RealmManager for each schema type in Kernel
+    # RealmManager registers own eventloop greenlet (update_kafka) with ProducerManager
 
     def __init__(self):
         # Start Signal Handlers
@@ -66,7 +66,7 @@ def __init__(self):
 
         # Clear objects and start
         self.kafka_status = KafkaStatus.SUBMISSION_PENDING
-        self.topic_managers = {}
+        self.realm_managers = {}
         self.run()
 
     def keep_alive_loop(self):
@@ -77,9 +77,9 @@ def keep_alive_loop(self):
     def run(self):
         self.threads = []
         self.threads.append(gevent.spawn(self.keep_alive_loop))
-        self.threads.append(gevent.spawn(self.check_schemas))
+        self.threads.append(gevent.spawn(self.check_realms))
         # Also going into this greenlet pool:
-        # Each TopicManager.update_kafka() from TopicManager.init
+        # Each RealmManager.update_kafka() from RealmManager.init
         gevent.joinall(self.threads)
 
     def kill(self, *args, **kwargs):
@@ -88,13 +88,21 @@ def kill(self, *args, **kwargs):
         self.http.stop()
         self.http.close()
         self.worker_pool.kill()
-        self.killed = True  # Flag checked by spawned TopicManagers to stop themselves
+        self.killed = True  # Flag checked by spawned RealmManagers to stop themselves
 
     def safe_sleep(self, dur):
         # keeps shutdown time low by yielding during sleep and checking if killed.
+        # limit sleep calls to prevent excess context switching that occurs on gevent.sleep
+        if dur < 5:
+            unit = 1
+        else:
+            res = dur % 5
+            dur = (dur - res) / 5
+            unit = 5
+            gevent.sleep(res)
         for x in range(int(dur)):
             if not self.killed:
-                gevent.sleep(1)
+                gevent.sleep(unit)
 
     # Connectivity
 
@@ -146,39 +154,29 @@ def init_db(self):
         init_offset_db()
         self.logger.info('OffsetDB initialized')
 
-    # Main Schema Loop
-    # Spawns TopicManagers for new schemas, updates schemas for workers on change.
-    def check_schemas(self):
-        # Checks for schemas in Kernel
-        # Creates TopicManagers for found schemas.
-        # Updates TopicManager.schema on schema change
+    # TODO swap over
+
+    # # main update loop
+    # # creates a manager / producer for each Realm
+    def check_realms(self):
         while not self.killed:
-            schemas = []
+            realms = []
             try:
-                schemas = self.kernel_client.get_schemas()
+                self.logger.debug('Checking for new realms')
+                realms = self.kernel_client.get_realms()
+                for realm in realms:
+                    if realm not in self.realm_managers.keys():
+                        self.logger.info(f'Realm connected: {realm}')
+                        self.realm_managers[realm] = RealmManager(self, realm)
+                if not realms:
+                    gevent.sleep(5)
+                else:
+                    gevent.sleep(30)
             except Exception as err:
                 self.logger.warning(f'No Kernel connection: {err}')
                 gevent.sleep(1)
                 continue
-
-            for schema in schemas:
-                name = schema['schema_name']
-                realm = schema['realm']
-                schema_name = f'{realm}.{name}'
-                if schema_name not in self.topic_managers.keys():
-                    self.logger.info(f'Topic connected: {schema_name}')
-                    self.topic_managers[schema_name] = TopicManager(self, schema, realm)
-                else:
-                    topic_manager = self.topic_managers[schema_name]
-                    if topic_manager.schema_changed(schema):
-                        topic_manager.update_schema(schema)
-                        self.logger.debug(f'Schema {schema_name} updated')
-                    else:
-                        self.logger.debug(f'Schema {schema_name} unchanged')
-
-            # Time between checks for schema change
-            self.safe_sleep(SETTINGS.get('sleep_time', 10))
-        self.logger.debug('No longer checking schemas')
+        self.logger.debug('No longer checking for new Realms')
 
     # Flask Functions
 
@@ -244,17 +242,21 @@ def request_status(self):
             'kafka_container_accessible': self.kafka_available(),
             'kafka_broker_information': self.broker_info(),
             'kafka_submission_status': str(self.kafka_status),  # This is just a status flag
-            'topics': {k: v.get_status() for k, v in self.topic_managers.items()},
+            'topics': {k: v.get_status() for k, v in self.realm_managers.items()},
         }
         with self.app.app_context():
             return jsonify(**status)
 
     @requires_auth
     def request_topics(self):
-        if not self.topic_managers:
+        if not self.realm_managers:
             return Response({})
 
-        status = {k: v.get_topic_size() for k, v in self.topic_managers.items()}
+        status = {}
+        for topic, manager in self.realm_managers.items():
+            status[topic] = {}
+            for name, sw in manager.schemas.items():
+                status[topic][name] = manager.get_topic_size(sw)
         with self.app.app_context():
             return jsonify(**status)
 
@@ -275,12 +277,18 @@ def request_rebuild(self):
     @requires_auth
     def handle_topic_command(self, request, status):
         topic = request.args.get('topic')
+        realm = request.args.get('realm')
+        if not realm:
+            return Response('A realm must be specified', 422)
         if not topic:
             return Response('A topic must be specified', 422)
-        if not self.topic_managers.get(topic):
-            return Response(f'Bad topic name {topic}', 422)
+        if not self.realm_managers.get(realm):
+            return Response(f'Bad realm name: {realm}', 422)
 
-        manager = self.topic_managers[topic]
+        manager = self.realm_managers[realm]
+        schema_wrapper = manager.schemas.get(topic)
+        if not schema_wrapper:
+            return Response(f'realm {realm} has no topic {topic}', 422)
         if status is TopicStatus.PAUSED:
             fn = manager.pause
         if status is TopicStatus.NORMAL:
@@ -289,7 +297,7 @@ def handle_topic_command(self, request, status):
             fn = manager.rebuild
 
         try:
-            res = fn()
+            res = fn(schema_wrapper)
             if not res:
                 return Response(f'Operation failed on {topic}', 500)
 
diff --git a/aether-producer/aether/producer/kernel.py b/aether-producer/aether/producer/kernel.py
index 8012211de..fe0987af1 100644
--- a/aether-producer/aether/producer/kernel.py
+++ b/aether-producer/aether/producer/kernel.py
@@ -33,7 +33,10 @@ def __init__(self):
         # last time kernel was checked for new updates
         self.last_check = None
         self.last_check_error = None
+        # limit number of messages in a single batch
         self.limit = int(SETTINGS.get('fetch_size', 100))
+        # send when message volume >= batch_size (kafka hard limit is 2MB)
+        self.batch_size = int(SETTINGS.get('publish_size', 100_000))
 
     def get_time_window_filter(self, query_time):
         # You can't always trust that a set from kernel made up of time window
@@ -61,6 +64,9 @@ def fn(row):
     def mode(self):
         raise NotImplementedError
 
+    def get_realms(self):
+        raise NotImplementedError
+
     def get_schemas(self):
         raise NotImplementedError
 
diff --git a/aether-producer/aether/producer/kernel_api.py b/aether-producer/aether/producer/kernel_api.py
index 777f927a9..57262382e 100644
--- a/aether-producer/aether/producer/kernel_api.py
+++ b/aether-producer/aether/producer/kernel_api.py
@@ -23,7 +23,7 @@
 
 from aether.producer.settings import SETTINGS
 from aether.producer.kernel import KernelClient, logger
-
+from aether.producer.utils import utf8size
 
 _REQUEST_ERROR_RETRIES = int(SETTINGS.get('request_error_retries', 3))
 
@@ -44,15 +44,23 @@
     '&page_size={page_size}'
     '&fields=id,schema,schema_name,schema_definition'
 )
-_ENTITIES_URL = (
+_ENTITIES_SINGLE_URL = (
     f'{_KERNEL_URL}/'
     'entities.json?'
     '&page_size={page_size}'
-    '&fields=id,modified,payload'
+    '&fields=id,modified,payload,schema'
     '&ordering=modified'
     '&modified__gt={modified}'
     '&schema={schema}'
 )
+_ENTITIES_ALL_URL = (
+    f'{_KERNEL_URL}/'
+    'entities.json?'
+    '&page_size={page_size}'
+    '&fields=id,modified,payload,schema'
+    '&ordering=modified'
+    '&modified__gt={modified}'
+)
 
 
 class KernelAPIClient(KernelClient):
@@ -60,12 +68,18 @@ class KernelAPIClient(KernelClient):
     def mode(self):
         return 'api'
 
-    def get_schemas(self):
+    def get_realms(self):
+        return self._fetch(url=_REALMS_URL)['realms']
+
+    def get_schemas(self, realm=None):
         self.last_check = datetime.now().isoformat()
 
         try:
             # get list of realms
-            realms = self._fetch(url=_REALMS_URL)['realms']
+            if not realm:
+                realms = self.get_realms()
+            else:
+                realms = [realm]
             for realm in realms:
                 # get list of schema decorators
                 _next_url = _SCHEMAS_URL.format(page_size=self.limit)
@@ -81,12 +95,18 @@ def get_schemas(self):
             logger.warning(self.last_check_error)
             return []
 
-    def check_updates(self, realm, schema_id, schema_name, modified):
-        url = _ENTITIES_URL.format(
-            page_size=1,
-            schema=schema_id,
-            modified=modified,
-        )
+    def check_updates(self, realm, schema_id=None, schema_name=None, modified=''):
+        if schema_id:
+            url = _ENTITIES_SINGLE_URL.format(
+                page_size=1,
+                schema=schema_id,
+                modified=modified or '',
+            )
+        else:
+            url = _ENTITIES_ALL_URL.format(
+                page_size=1,
+                modified=modified or '',
+            )
         try:
             response = self._fetch(url=url, realm=realm)
             return response['count'] > 1
@@ -94,37 +114,59 @@ def check_updates(self, realm, schema_id, schema_name, modified):
             logger.warning('Could not access kernel API to look for updates')
             return False
 
-    def count_updates(self, realm, schema_id, schema_name, modified=''):
-        url = _ENTITIES_URL.format(
-            page_size=1,
-            schema=schema_id,
-            modified=modified or '',
-        )
+    def count_updates(self, realm, schema_id=None, schema_name=None, modified=''):
+        if schema_id:
+            url = _ENTITIES_SINGLE_URL.format(
+                page_size=1,
+                schema=schema_id,
+                modified=modified or '',
+            )
+        else:
+            url = _ENTITIES_ALL_URL.format(
+                page_size=1,
+                modified=modified or '',
+            )
         try:
             _count = self._fetch(url=url, realm=realm)['count']
-            logger.debug(f'Reporting requested size for {schema_name} of {_count}')
+            logger.debug(
+                f'Reporting requested size for {schema_name or "all entities"} of {_count}')
             return {'count': _count}
         except Exception:
             logger.warning('Could not access kernel API to look for updates')
             return -1
 
-    def get_updates(self, realm, schema_id, schema_name, modified):
-        url = _ENTITIES_URL.format(
-            page_size=self.limit,
-            schema=schema_id,
-            modified=modified,
-        )
+    def get_updates(self, realm, schema_id=None, schema_name=None, modified=''):
+        if schema_id:
+            url = _ENTITIES_SINGLE_URL.format(
+                page_size=self.limit,
+                schema=schema_id,
+                modified=modified or '',
+            )
+        else:
+            url = _ENTITIES_ALL_URL.format(
+                page_size=self.limit,
+                modified=modified or '',
+            )
 
         try:
             query_time = datetime.now()
             window_filter = self.get_time_window_filter(query_time)
 
             response = self._fetch(url=url, realm=realm)
-            return [
-                entry
-                for entry in response['results']
-                if window_filter(entry)
-            ]
+            res = []
+            size = 0
+            for entry in response['results']:
+                if window_filter(entry):
+                    new_size = size + utf8size(entry)
+                    res.append(entry)
+                    if new_size >= self.batch_size:
+                        # when we get over the batch size, truncate
+                        # this means even with a batch size of 1, if a message
+                        # is 10, we still emit one message
+                        return res
+                    size = new_size
+
+            return res
 
         except Exception:
             logger.warning('Could not access kernel API to look for updates')
diff --git a/aether-producer/aether/producer/kernel_db.py b/aether-producer/aether/producer/kernel_db.py
index d70aac69c..bc767b7ea 100644
--- a/aether-producer/aether/producer/kernel_db.py
+++ b/aether-producer/aether/producer/kernel_db.py
@@ -33,42 +33,76 @@
 from aether.producer.db import PriorityDatabasePool
 from aether.producer.settings import SETTINGS
 from aether.producer.kernel import KernelClient, logger
+from aether.producer.utils import utf8size
 
 
-_SCHEMAS_SQL = '''
+_REALMS_SQL = '''
+    SELECT DISTINCT ON (realm) realm
+      FROM kernel_schema_vw
+'''
+
+_SCHEMAS_SQL_ALL_REALMS = '''
     SELECT schema_id, schema_name, schema_definition, realm
       FROM kernel_schema_vw
 '''
 
-_CHECK_UPDATES_SQL = '''
+_SCHEMAS_SQL_SINGLE_REALM = _SCHEMAS_SQL_ALL_REALMS + '''
+     WHERE realm     = {realm};'''
+
+
+__CHECK_UPDATES_SQL = '''
     SELECT id
       FROM kernel_entity_vw
      WHERE modified  > {modified}
-       AND schema_id = {schema}
        AND realm     = {realm}
+'''
+
+_CHECK_UPDATES_SQL_SINGLE = __CHECK_UPDATES_SQL + '''
+       AND schema_id = {schema}
      LIMIT 1;
 '''
 
-_COUNT_UPDATES_SQL = '''
+_CHECK_UPDATES_SQL_ALL = __CHECK_UPDATES_SQL + '''
+     LIMIT 1;
+'''
+
+__COUNT_UPDATES_SQL = '''
     SELECT COUNT(id)
       FROM kernel_entity_vw
-     WHERE schema_id = {schema}
-       AND realm     = {realm};
+     WHERE realm     = {realm}'''
+
+_COUNT_UPDATES_SQL_SINGLE = __COUNT_UPDATES_SQL + '''
+       AND schema_id = {schema};
 '''
-_COUNT_MODIFIED_UPDATES_SQL = '''
+
+_COUNT_UPDATES_SQL_ALL = __COUNT_UPDATES_SQL + ';'
+
+__COUNT_MODIFIED_UPDATES_SQL = '''
     SELECT COUNT(id)
       FROM kernel_entity_vw
      WHERE modified  > {modified}
-       AND schema_id = {schema}
-       AND realm     = {realm};
+       AND realm     = {realm}'''
+
+_COUNT_MODIFIED_UPDATES_SQL_ALL = __COUNT_MODIFIED_UPDATES_SQL + ';'
+
+_COUNT_MODIFIED_UPDATES_SQL_SINGLE = __COUNT_MODIFIED_UPDATES_SQL + '''
+       AND schema_id = {schema};
 '''
 
-_GET_UPDATES_SQL = '''
+__GET_UPDATES_SQL = '''
     SELECT *
       FROM kernel_entity_vw
      WHERE modified  > {modified}
-       AND schema_id = {schema}
        AND realm     = {realm}
+'''
+
+_GET_UPDATES_SQL_ALL = __GET_UPDATES_SQL + '''
+     ORDER BY modified ASC
+     LIMIT {limit};
+'''
+
+_GET_UPDATES_SQL_SINGLE = __GET_UPDATES_SQL + '''
+       AND schema_id = {schema}
      ORDER BY modified ASC
      LIMIT {limit};
 '''
@@ -87,10 +121,18 @@ def __init__(self):
     def mode(self):
         return 'db'
 
-    def get_schemas(self):
+    def get_realms(self):
+        query = sql.SQL(_REALMS_SQL)
+        cursor = self._exec_sql('get_realms', 1, query)
+        return [row['realm'] for row in cursor]
+
+    def get_schemas(self, realm=None):
         self.last_check = datetime.now().isoformat()
         name = 'schemas_query'
-        query = sql.SQL(_SCHEMAS_SQL)
+        if realm:
+            query = sql.SQL(_SCHEMAS_SQL_SINGLE_REALM).format(realm=sql.Literal(realm))
+        else:
+            query = sql.SQL(_SCHEMAS_SQL_ALL_REALMS)
         cursor = self._exec_sql(name, 1, query)
         if cursor:
             self.last_check_error = None
@@ -101,57 +143,94 @@ def get_schemas(self):
             logger.warning('Could not access kernel database to get topics')
             return []
 
-    def check_updates(self, realm, schema_id, schema_name, modified):
-        query = sql.SQL(_CHECK_UPDATES_SQL).format(
-            modified=sql.Literal(modified),
-            schema=sql.Literal(schema_id),
-            realm=sql.Literal(realm),
-        )
-        cursor = self._exec_sql(schema_name, 1, query)
+    def check_updates(self, realm, schema_id=None, schema_name=None, modified=''):
+        if schema_id:
+            query = sql.SQL(_CHECK_UPDATES_SQL_SINGLE).format(
+                modified=sql.Literal(modified),
+                schema=sql.Literal(schema_id),
+                realm=sql.Literal(realm),
+            )
+        else:
+            query = sql.SQL(_CHECK_UPDATES_SQL_ALL).format(
+                modified=sql.Literal(modified),
+                realm=sql.Literal(realm),
+            )
+        cursor = self._exec_sql(schema_name or 'All', 1, query)
         if cursor:
             return sum([1 for i in cursor]) > 0
         else:
             logger.warning('Could not access kernel database to look for updates')
             return False
 
-    def count_updates(self, realm, schema_id, schema_name, modified=''):
+    def count_updates(self, realm, schema_id=None, schema_name=None, modified=''):
         if modified:
-            query = sql.SQL(_COUNT_MODIFIED_UPDATES_SQL).format(
-                modified=sql.Literal(modified),
-                schema=sql.Literal(schema_id),
-                realm=sql.Literal(realm),
-            )
+            if schema_id:
+                query = sql.SQL(_COUNT_MODIFIED_UPDATES_SQL_SINGLE).format(
+                    modified=sql.Literal(modified),
+                    schema=sql.Literal(schema_id),
+                    realm=sql.Literal(realm),
+                )
+            else:
+                query = sql.SQL(_COUNT_MODIFIED_UPDATES_SQL_ALL).format(
+                    modified=sql.Literal(modified),
+                    realm=sql.Literal(realm),
+                )
+
         else:
-            query = sql.SQL(_COUNT_UPDATES_SQL).format(
-                schema=sql.Literal(schema_id),
-                realm=sql.Literal(realm),
-            )
-        cursor = self._exec_sql(schema_name, 0, query)
+            if schema_id:
+                query = sql.SQL(_COUNT_UPDATES_SQL_SINGLE).format(
+                    schema=sql.Literal(schema_id),
+                    realm=sql.Literal(realm),
+                )
+            else:
+                query = sql.SQL(_COUNT_UPDATES_SQL_ALL).format(
+                    realm=sql.Literal(realm),
+                )
+        cursor = self._exec_sql(schema_name or 'All', 0, query)
         if cursor:
             _count = cursor.fetchone()[0]
-            logger.debug(f'Reporting requested size for {schema_name} of {_count}')
+            logger.debug(f'Reporting requested size for {schema_name or "All"} of {_count}')
             return {'count': _count}
         else:
             logger.warning('Could not access kernel database to look for updates')
             return -1
 
-    def get_updates(self, realm, schema_id, schema_name, modified):
-        query = sql.SQL(_GET_UPDATES_SQL).format(
-            modified=sql.Literal(modified),
-            schema=sql.Literal(schema_id),
-            realm=sql.Literal(realm),
-            limit=sql.Literal(self.limit),
-        )
+    def get_updates(self, realm, schema_id=None, schema_name=None, modified=''):
+        if schema_id:
+            query = sql.SQL(_GET_UPDATES_SQL_SINGLE).format(
+                modified=sql.Literal(modified),
+                schema=sql.Literal(schema_id),
+                realm=sql.Literal(realm),
+                limit=sql.Literal(self.limit),
+            )
+        else:
+            query = sql.SQL(_GET_UPDATES_SQL_ALL).format(
+                modified=sql.Literal(modified),
+                realm=sql.Literal(realm),
+                limit=sql.Literal(self.limit),
+            )
 
         query_time = datetime.now()
-        cursor = self._exec_sql(schema_name, 2, query)
+        cursor = self._exec_sql(schema_name or 'All', 2, query)
         if cursor:
             window_filter = self.get_time_window_filter(query_time)
-            return [
-                {key: row[key] for key in row.keys()}
-                for row in cursor
-                if window_filter(row)
-            ]
+
+            res = []
+            size = 0
+            for row in cursor:
+                if window_filter(row):
+                    entry = {key: row[key] for key in row.keys()}
+                    new_size = size + utf8size(entry)
+                    res.append(entry)
+                    if new_size >= self.batch_size:
+                        # when we get over the batch size, truncate
+                        # this means even with a batch size of 1, if a message
+                        # is 10, we still emit one message
+                        return res
+                    size = new_size
+
+            return res
+
         else:
             logger.warning('Could not access kernel database to look for updates')
             return []
diff --git a/aether-producer/aether/producer/topic.py b/aether-producer/aether/producer/topic.py
index ba4a7b4e6..a2c9fc37a 100644
--- a/aether-producer/aether/producer/topic.py
+++ b/aether-producer/aether/producer/topic.py
@@ -17,13 +17,12 @@
 # under the License.
 
 import ast
-import concurrent
 import enum
 import gevent
 import io
 import json
-import sys
 import traceback
+from typing import (Any, Dict)
 from datetime import datetime
 
 from confluent_kafka import Producer
@@ -40,6 +39,53 @@
 logger = get_logger('producer-topic')
 
 
+class SchemaWrapper(object):
+    definition: Dict[str, Any]
+    name: str
+    offset: str  # current offset
+    schema: spavro.schema.Schema
+    schema_id: str
+    realm: str
+    topic: str
+    operating_status: 'TopicStatus'
+
+    def __init__(self, realm, name=None, schema_id=None, definition=None, aether_definition=None):
+        self.realm = realm
+        if aether_definition:
+            self.name = aether_definition['schema_name']
+            self.schema_id = aether_definition['schema_id']
+        elif all([name, schema_id]):
+            self.name = name
+            self.schema_id = schema_id
+        else:
+            raise RuntimeError('Requires either name and id, or aether_definition')
+        if any([definition, aether_definition]):
+            self.update(aether_definition, definition)
+        self.get_topic_name()
+        self.operating_status = TopicStatus.NORMAL
+
+    def definition_from_aether(self, aether_definition: Dict[str, Any]):
+        return ast.literal_eval(json.dumps(aether_definition['schema_definition']))
+
+    def is_equal(self, new_aether_definition) -> bool:
+        return (
+            json.dumps(self.definition_from_aether(new_aether_definition))
+            == json.dumps(self.definition))
+
+    def update(self, aether_definition: Dict[str, Any] = None, definition: Dict[str, Any] = None):
+        if not any([aether_definition, definition]):
+            raise RuntimeError('Expected one of [definition, aether_definition]')
+        if aether_definition:
+            self.definition = self.definition_from_aether(aether_definition)
+        elif definition:
+            self.definition = definition
+        self.schema = spavro.schema.parse(json.dumps(self.definition))
+
+    def get_topic_name(self):
+        topic_base = SETTINGS.get('topic_settings', {}).get('name_modifier', '%s') % self.name
+        self.topic = f'{self.realm}.{topic_base}'
+
+
 class KafkaStatus(enum.Enum):
     SUBMISSION_PENDING = 1
     SUBMISSION_FAILURE = 2
@@ -55,396 +101,324 @@ class TopicStatus(enum.Enum):
     ERROR = 5
 
 
-class TopicManager(object):
+class RealmManager(object):
 
-    # Creates a long running job on TopicManager.update_kafka
+    # Creates a long running job on RealmManager.update_kafka
 
-    def __init__(self, context, schema, realm):
+    def __init__(self, context, realm):
         self.context = context
-        self.pk = schema['schema_id']
-        self.name = schema['schema_name']
         self.realm = realm
-        self.offset = ''
-        self.operating_status = TopicStatus.INITIALIZING
-        self.status = {
-            'last_errors_set': {},
-            'last_changeset_status': {}
-        }
-        self.change_set = {}
-        self.successful_changes = []
-        self.failed_changes = {}
         self.sleep_time = int(SETTINGS.get('sleep_time', 10))
         self.window_size_sec = int(SETTINGS.get('window_size_sec', 3))
-
         self.kafka_failure_wait_time = float(SETTINGS.get('kafka_failure_wait_time', 10))
-
-        try:
-            topic_base = SETTINGS.get('topic_settings', {}).get('name_modifier', '%s') % self.name
-            self.topic_name = f'{self.realm}.{topic_base}'
-        except Exception:  # Bad Name
-            logger.critical((f'invalid name_modifier using topic name for topic: {self.name}.'
-                             ' Update configuration for topic_settings.name_modifier'))
-            # This is a failure which could cause topics to collide. We'll kill the producer
-            # so the configuration can be updated.
-            sys.exit(1)
-
-        self.update_schema(schema)
+        self.status = {}
+        self.schemas = {}
+        self.known_topics = []
         self.get_producer()
         # Spawn worker and give to pool.
-        logger.debug(f'Spawning kafka update thread: {self.topic_name}')
-        self.context.threads.append(gevent.spawn(self.update_kafka))
-        logger.debug(f'Checking for existence of topic {self.topic_name}')
-        while not self.check_topic():
-            if self.create_topic():
-                break
-            logger.debug(f'Waiting 30 seconds to retry creation of topic {self.topic_name}')
-            self.context.safe_sleep(30)
-        self.operating_status = TopicStatus.NORMAL
-
-    def check_topic(self):
-        topics = [t for t in self.producer.list_topics().topics.keys()]
-        if self.topic_name in topics:
-            logger.debug(f'Topic {self.topic_name} already exists.')
-            return True
-
-        logger.debug(f'Topic {self.name} does not exist. current topics: {topics}')
-        return False
-
-    def create_topic(self):
-        logger.debug(f'Trying to create topic {self.topic_name}')
-
-        kadmin = self.context.kafka_admin_client
-        topic_config = SETTINGS.get('kafka_settings', {}).get('default.topic.config')
-        partitions = int(SETTINGS.get('kafka_default_topic_partitions', 1))
-        replicas = int(SETTINGS.get('kafka_default_topic_replicas', 1))
-        topic = NewTopic(
-            self.topic_name,
-            num_partitions=partitions,
-            replication_factor=replicas,
-            config=topic_config,
-        )
-        fs = kadmin.create_topics([topic])
-        # future must return before timeout
-        for f in concurrent.futures.as_completed(iter(fs.values()), timeout=60):
-            e = f.exception()
-            if not e:
-                logger.info(f'Created topic {self.name}')
-                return True
-            else:
-                logger.warning(f'Topic {self.name} could not be created: {e}')
-                return False
+        self.context.threads.append(gevent.spawn(self.update_loop))
+
+    def update_topics(self):
+        self.known_topics = [t for t in self.producer.list_topics().topics.keys()]
+
+    def create_topic(self, topic=None, topics=None):
+        topic_objects = []
+        if not topics:
+            topics = [topic]
+        for t in topics:
+            logger.debug(f'Trying to create topic {t}')
+
+            kadmin = self.context.kafka_admin_client
+            topic_config = SETTINGS.get('kafka_settings', {}).get('default.topic.config')
+            partitions = int(SETTINGS.get('kafka_default_topic_partitions', 1))
+            replicas = int(SETTINGS.get('kafka_default_topic_replicas', 1))
+            topic_objects.append(
+                NewTopic(
+                    t,
+                    num_partitions=partitions,
+                    replication_factor=replicas,
+                    config=topic_config,
+                )
+            )
+        kadmin.create_topics(topic_objects)
 
     def get_producer(self):
         self.producer = Producer(**KAFKA_SETTINGS)
-        logger.debug(f'Producer for {self.name} started...')
+        logger.debug(f'Producer for {self.realm} started...')
+
+    # API for realm status
+
+    def get_status(self):
+        # Updates inflight status and returns to Flask called
+        for sw in self.schemas.values():
+            self.status[sw.name]['operating_status'] = str(sw.operating_status)
+        return self.status
 
     # API Calls to Control Topic
 
-    def pause(self):
+    def pause(self, sw: SchemaWrapper):
         # Stops sending of data on this topic until resume is called or Producer restarts.
-        if self.operating_status is not TopicStatus.NORMAL:
-            logger.info(f'Topic {self.name} could not pause, status: {self.operating_status}.')
+        if sw.operating_status is not TopicStatus.NORMAL:
+            logger.info(f'Topic {sw.name} could not pause, status: {sw.operating_status}.')
             return False
 
-        logger.info(f'Topic {self.name} is pausing.')
-        self.operating_status = TopicStatus.PAUSED
+        logger.info(f'Topic {sw.name} is pausing.')
+        sw.operating_status = TopicStatus.PAUSED
         return True
 
-    def resume(self):
+    def resume(self, sw: SchemaWrapper):
         # Resume sending data after pausing.
-        if self.operating_status is not TopicStatus.PAUSED:
-            logger.info(f'Topic {self.name} could not resume, status: {self.operating_status}.')
+        if sw.operating_status is not TopicStatus.PAUSED:
+            logger.info(f'Topic {sw.name} could not resume, status: {sw.operating_status}.')
             return False
 
-        logger.info(f'Topic {self.name} is resuming.')
-        self.operating_status = TopicStatus.NORMAL
+        logger.info(f'Topic {sw.name} is resuming.')
+        sw.operating_status = TopicStatus.NORMAL
         return True
 
-    # Functions to rebuilt this topic
+    # Functions to rebuild this topic
 
-    def rebuild(self):
+    def rebuild(self, sw: SchemaWrapper):
         # API Call
-        logger.warn(f'Topic {self.name} is being REBUIT!')
+        logger.warn(f'Topic {sw.name} is being REBUIT!')
         # kick off rebuild process
-        self.context.threads.append(gevent.spawn(self.handle_rebuild))
+        _fn = self._make_rebuild_process(sw)
+        self.context.threads.append(gevent.spawn(_fn))
         return True
 
-    def handle_rebuild(self):
-        # greened background task to handle rebuilding of topic
-        self.operating_status = TopicStatus.REBUILDING
-        tag = f'REBUILDING {self.name}:'
-        sleep_time = self.sleep_time * 1.5
-        logger.info(f'{tag} waiting {sleep_time}(sec) for inflight ops to resolve')
-        self.context.safe_sleep(sleep_time)
-        logger.info(f'{tag} Deleting Topic')
-        self.producer = None
-
-        if not self.delete_this_topic():
-            logger.warning(f'{tag} FAILED. Topic will not resume.')
-            self.operating_status = TopicStatus.LOCKED
-            return
+    def _make_rebuild_process(self, sw: SchemaWrapper):
 
-        logger.warn(f'{tag} Resetting Offset.')
-        self.set_offset('')
-        logger.info(f'{tag} Rebuilding Topic Producer')
-        self.producer = Producer(**KAFKA_SETTINGS)
-        logger.warn(f'{tag} Wipe Complete. /resume to complete operation.')
-        self.operating_status = TopicStatus.PAUSED
+        def fn():
+            # greened background task to handle rebuilding of topic
+            sw.operating_status = TopicStatus.REBUILDING
+            tag = f'REBUILDING {sw.topic}:'
+            sleep_time = self.sleep_time * 1.5
+            logger.info(f'{tag} waiting {sleep_time}(sec) for inflight ops to resolve')
+            self.context.safe_sleep(sleep_time)
+            logger.info(f'{tag} Deleting Topic')
+
+            if not self.delete_this_topic(sw):
+                logger.warning(f'{tag} FAILED. Topic will not resume.')
+                sw.operating_status = TopicStatus.LOCKED
+                return
 
-    def delete_this_topic(self):
+            logger.warn(f'{tag} Resetting Offset.')
+            self.set_offset('', sw)
+            logger.info(f'{tag} Rebuilding Topic Producer')
+            logger.warn(f'{tag} Wipe Complete. /resume to complete operation.')
+            sw.operating_status = TopicStatus.PAUSED
+        return fn
+
+    def delete_this_topic(self, sw):
         kadmin = self.context.kafka_admin_client
-        fs = kadmin.delete_topics([self.name], operation_timeout=60)
-        future = fs.get(self.name)
+        fs = kadmin.delete_topics([sw.topic], operation_timeout=60)
+        future = fs.get(sw.topic)
         for x in range(60):
             if not future.done():
                 if (x % 5 == 0):
-                    logger.debug(f'REBUILDING {self.name}: Waiting for future to complete')
+                    logger.debug(f'REBUILDING {sw.topic}: Waiting for future to complete')
                 gevent.sleep(1)
             else:
                 return True
         return False
 
-    def updates_available(self):
-        return self.context.kernel_client.check_updates(self.realm, self.pk, self.name, self.offset)
+    # Get Data for a Topic from Kernel
 
-    def get_db_updates(self):
-        return self.context.kernel_client.get_updates(self.realm, self.pk, self.name, self.offset)
+    def updates_available(self, sw: SchemaWrapper):
+        return self.context.kernel_client.check_updates(sw.realm, sw.schema_id, sw.name, sw.offset)
 
-    def get_topic_size(self):
-        return self.context.kernel_client.count_updates(self.realm, self.pk, self.name)
+    def get_db_updates(self, sw: SchemaWrapper):
+        return self.context.kernel_client.get_updates(sw.realm, sw.schema_id, sw.name, sw.offset)
 
-    def update_schema(self, schema_obj):
-        self.schema_obj = self.parse_schema(schema_obj)
-        self.schema = spavro.schema.parse(json.dumps(self.schema_obj))
+    def get_topic_size(self, sw: SchemaWrapper):
+        return self.context.kernel_client.count_updates(sw.realm, sw.schema_id, sw.name)
 
-    def parse_schema(self, schema_obj):
-        # We split this method from update_schema because schema_obj as it is can not
-        # be compared for differences. literal_eval fixes this. As such, this is used
-        # by the schema_changed() method.
-        # schema_obj is a nested OrderedDict, which needs to be stringified
-        return ast.literal_eval(json.dumps(schema_obj['schema_definition']))
+    # Updates
 
-    def schema_changed(self, schema_candidate):
-        # for use by ProducerManager.check_schemas()
-        return self.parse_schema(schema_candidate) != self.schema_obj
+    # # Main Loop
 
-    def get_status(self):
-        # Updates inflight status and returns to Flask called
-        self.status['operating_status'] = str(self.operating_status)
-        self.status['inflight'] = [i for i in self.change_set.keys()]
-        return self.status
+    def update_loop(self):
+        while not self.context.killed:
+            logger.info(f'Looking for updates on: {self.realm}')
+            self.producer.poll(0)
+            self.update_schemas()
+            res = 0
+            for sw in self.schemas.values():
+                res += self.update_kafka(sw) or 0
+            if res:
+                self.context.safe_sleep(1)  # yield instead of waiting for flush
+                self.producer.flush(timeout=20)
+            else:
+                logger.info(f'No updates on: {self.realm}')
+            self.context.safe_sleep(self.sleep_time)  # wait for next batch
+
+    # # Schema Update
+
+    def update_schemas(self):
+        schemas = self.context.kernel_client.get_schemas(realm=self.realm)
+        self.update_topics()
+        new_topics = []
+        for aether_definition in schemas:
+            schema_name = aether_definition['schema_name']
+            if schema_name not in self.schemas:
+                self.schemas[schema_name] = SchemaWrapper(
+                    self.realm, aether_definition=aether_definition
+                )
+                self.status[schema_name] = {}
+                topic = self.schemas[schema_name].topic
+                if topic not in self.known_topics:
+                    new_topics.append(topic)
 
-    # Callback function registered with Kafka Producer to acknowledge receipt
-    def kafka_callback(self, err=None, msg=None, _=None, **kwargs):
-        if err:
-            logger.warning(f'ERROR [{err}, {msg}, {kwargs}]')
+            else:
+                if not self.schemas[schema_name].is_equal(aether_definition):
+                    self.schemas[schema_name].update(aether_definition=aether_definition)
+        if new_topics:
+            self.create_topic(topics=new_topics)
 
-        with io.BytesIO() as obj:
-            obj.write(msg.value())
-            reader = DataFileReader(obj, DatumReader())
-            for message in reader:
-                _id = message.get('id')
-                if err:
-                    logger.debug(f'NO-SAVE: {_id} in topic {self.topic_name} | err {err.name()}')
-                    self.failed_changes[_id] = err
-                else:
-                    logger.debug(f'SAVE: {_id} in topic {self.topic_name}')
-                    self.successful_changes.append(_id)
-
-    def update_kafka(self):
+    # # Kafka Publish
+
+    def update_kafka(self, sw: SchemaWrapper):
         # Main update loop
-        # Monitors postgres for changes via TopicManager.updates_available
-        # Consumes updates to the Postgres DB via TopicManager.get_db_updates
+        # Monitors postgres for changes via RealmManager.updates_available
+        # Consumes updates to the Postgres DB via RealmManager.get_db_updates
         # Sends new messages to Kafka
-        # Registers message callback (ok or fail) to TopicManager.kafka_callback
-        # Waits for all messages to be accepted or timeout in TopicManager.wait_for_kafka
-        logger.debug(f'Topic {self.name}: Initializing')
+        # Registers message callback (ok or fail) to RealmManager.kafka_callback
+        logger.debug(f'Checking {sw.topic}')
+
+        if sw.operating_status is TopicStatus.INITIALIZING:
+            logger.debug(f'Waiting for topic {sw.topic} to initialize...')
+            return
+
+        if sw.operating_status is not TopicStatus.NORMAL:
+            logger.debug(
+                f'Topic {sw.topic} not updating, status: {sw.operating_status}'
+                f', waiting {self.sleep_time}(sec)')
+            return
+
+        if not self.context.kafka_available():
+            logger.debug('Kafka Container not accessible, waiting.')
+            return
+
+        sw.offset = self.get_offset(sw) or ''
+
+        if not self.updates_available(sw):
+            logger.debug(f'No updates on {sw.topic}')
+            return
 
-        while self.operating_status is TopicStatus.INITIALIZING:
-            if self.context.killed:
+        try:
+            logger.debug(f'Getting Changeset for {sw.topic}')
+            new_rows = self.get_db_updates(sw)
+            if not new_rows:
+                logger.debug(f'No changes on {sw.topic}')
                 return
-            logger.debug(f'Waiting for topic {self.name} to initialize...')
-            self.context.safe_sleep(self.sleep_time)
-            pass
+            end_offset = new_rows[-1].get('modified')
+        except Exception as pge:
+            logger.warning(f'Could not get new records from kernel: {pge}')
+            return
 
-        while not self.context.killed:
-            if self.operating_status is not TopicStatus.NORMAL:
-                logger.debug(
-                    f'Topic {self.name} not updating, status: {self.operating_status}'
-                    f', waiting {self.sleep_time}(sec)')
-                self.context.safe_sleep(self.sleep_time)
-                continue
-
-            if not self.context.kafka_available():
-                logger.debug('Kafka Container not accessible, waiting.')
-                self.context.safe_sleep(self.sleep_time)
-                continue
-
-            self.offset = self.get_offset() or ''
-            if not self.updates_available():
-                logger.debug('No updates')
-                self.context.safe_sleep(self.sleep_time)
-                continue
-
-            try:
-                logger.debug(f'Getting Changeset for {self.name}')
-                self.change_set = {}
-                new_rows = self.get_db_updates()
-                if not new_rows:
-                    self.context.safe_sleep(self.sleep_time)
-                    continue
-
-                end_offset = new_rows[-1].get('modified')
-            except Exception as pge:
-                logger.warning(f'Could not get new records from kernel: {pge}')
-                self.context.safe_sleep(self.sleep_time)
-                continue
-
-            try:
-                with io.BytesIO() as bytes_writer:
-                    writer = DataFileWriter(
-                        bytes_writer, DatumWriter(), self.schema, codec='deflate')
-
-                    for row in new_rows:
-                        _id = row['id']
-                        msg = row.get('payload')
-                        modified = row.get('modified')
-                        if validate(self.schema, msg):
-                            # Message validates against current schema
-                            logger.debug(
-                                f'ENQUEUE MSG TOPIC: {self.name}, ID: {_id}, MOD: {modified}')
-                            self.change_set[_id] = row
-                            writer.append(msg)
-                        else:
-                            # Message doesn't have the proper format for the current schema.
-                            logger.warning(
-                                f'SCHEMA_MISMATCH: NOT SAVED! TOPIC: {self.name}, ID: {_id}')
-
-                    writer.flush()
-                    raw_bytes = bytes_writer.getvalue()
-
-                self.producer.poll(0)
-                self.producer.produce(
-                    self.topic_name,
-                    raw_bytes,
-                    callback=self.kafka_callback
-                )
-                self.producer.flush()
-                self.wait_for_kafka(end_offset, failure_wait_time=self.kafka_failure_wait_time)
-
-            except Exception as ke:
-                logger.warning(f'error in Kafka save: {ke}')
-                logger.warning(traceback.format_exc())
-                self.context.safe_sleep(self.sleep_time)
-
-    def wait_for_kafka(self, end_offset, timeout=10, iters_per_sec=10, failure_wait_time=10):
-        # Waits for confirmation of message receipt from Kafka before moving to next changeset
-        # Logs errors and status to log and to web interface
-
-        sleep_time = timeout / (timeout * iters_per_sec)
-        change_set_size = len(self.change_set)
-        errors = {}
-        for i in range(timeout * iters_per_sec):
-            # whole changeset failed; systemic failure likely; sleep it off and try again
-            if len(self.failed_changes) >= change_set_size:
-                self.handle_kafka_errors(change_set_size, True, failure_wait_time)
-                self.clear_changeset()
-                logger.info(
-                    f'Changeset not saved; likely broker outage, sleeping worker for {self.name}')
-                self.context.safe_sleep(failure_wait_time)
-                return  # all failed; ignore changeset
-
-            # All changes were saved
-            elif len(self.successful_changes) == change_set_size:
-                logger.debug(f'All changes saved ok in topic {self.name}.')
-                break
-
-            # Remove successful and failed changes
-            for k in self.failed_changes:
-                try:
-                    del self.change_set[k]
-                except KeyError:
-                    pass  # could have been removed on previous iter
-
-            for k in self.successful_changes:
-                try:
-                    del self.change_set[k]
-                except KeyError:
-                    pass  # could have been removed on previous iter
-
-            # All changes registered
-            if len(self.change_set) == 0:
-                break
-
-            gevent.sleep(sleep_time)
-
-        # Timeout reached or all messages returned ( and not all failed )
-
-        self.status['last_changeset_status'] = {
-            'changes': change_set_size,
-            'failed': len(self.failed_changes),
-            'succeeded': len(self.successful_changes),
+        try:
+            with io.BytesIO() as bytes_writer:
+                writer = DataFileWriter(
+                    bytes_writer, DatumWriter(), sw.schema, codec='deflate')
+
+                for row in new_rows:
+                    _id = row['id']
+                    msg = row.get('payload')
+                    modified = row.get('modified')
+                    if validate(sw.schema, msg):
+                        # Message validates against current schema
+                        logger.debug(
+                            f'ENQUEUE MSG TOPIC: {sw.topic}, ID: {_id}, MOD: {modified}')
+                        writer.append(msg)
+                    else:
+                        # Message doesn't have the proper format for the current schema.
+                        logger.warning(
+                            f'SCHEMA_MISMATCH: NOT SAVED! TOPIC: {sw.topic}, ID: {_id}')
+
+                writer.flush()
+                raw_bytes = bytes_writer.getvalue()
+
+            self.producer.produce(
+                sw.topic,
+                raw_bytes,
+                callback=self._make_kafka_callback(sw, end_offset)
+            )
+            return len(new_rows)
+
+        except Exception as ke:
+            logger.warning(f'error in Kafka save: {ke}')
+            logger.warning(traceback.format_exc())
+
+    # Callbacks
+
+    def _make_kafka_callback(self, sw: SchemaWrapper, end_offset):
+
+        def _callback(err=None, msg=None, _=None, **kwargs):
+            if err:
+                logger.warning(f'ERROR [{err}, {msg}, {kwargs}]')
+                return self._kafka_failed(sw, err, msg)
+            return self._kafka_ok(sw, end_offset, msg)
+
+        return _callback
+
+    def _kafka_ok(self, sw: SchemaWrapper, end_offset, msg):
+        _change_size = 0
+        with io.BytesIO() as obj:
+            obj.write(msg.value())
+            reader = DataFileReader(obj, DatumReader())
+            _change_size = sum([1 for i in reader])
+            logger.info(f'saved {_change_size} messages in topic {sw.topic}. Offset: {end_offset}')
+        self.set_offset(end_offset, sw)
+        self.status[sw.name]['last_changeset_status'] = {
+            'changes': _change_size,
+            'failed': 0,
+            'succeeded': _change_size,
             'timestamp': datetime.now().isoformat(),
         }
-        if errors:
-            self.handle_kafka_errors(change_set_size, all_failed=False)
-        self.clear_changeset()
-        # Once we're satisfied, we set the new offset past the processed messages
-        self.context.kafka_status = KafkaStatus.SUBMISSION_SUCCESS
-        self.set_offset(end_offset)
-        # Sleep so that elements passed in the current window become eligible
-        self.context.safe_sleep(self.window_size_sec)
-
-    def handle_kafka_errors(self, change_set_size, all_failed=False, failure_wait_time=10):
-        # Errors in saving data to Kafka are handled and logged here
-        errors = {}
-        for _id, err in self.failed_changes.items():
-            # accumulate error types
-            error_type = str(err.name())
-            errors[error_type] = errors.get(error_type, 0) + 1
 
+    def _kafka_failed(self, sw: SchemaWrapper, err, msg):
+        _change_size = 0
+        with io.BytesIO() as obj:
+            obj.write(msg.value())
+            reader = DataFileReader(obj, DatumReader())
+            for message in reader:
+                _change_size += 1
+                _id = message.get('id')
+                logger.debug(f'NO-SAVE: {_id} in topic {sw.topic} | err {err.name()}')
         last_error_set = {
-            'changes': change_set_size,
-            'errors': errors,
+            'changes': _change_size,
+            'failed': _change_size,
+            'succeeded': 0,
+            'errors': str(err),
             'outcome': 'RETRY',
             'timestamp': datetime.now().isoformat(),
         }
+        self.status[sw.name]['last_errors_set'] = last_error_set
+        self.context.kafka_status = KafkaStatus.SUBMISSION_FAILURE
 
-        if not all_failed:
-            # Collect Error types for reporting
-            for _id, err in self.failed_changes.items():
-                logger.warning(f'PRODUCER_FAILURE: T: {self.name} ID {_id}, ERR_MSG {err.name()}')
-
-            dropped_messages = change_set_size - len(self.successful_changes)
-            errors['NO_REPLY'] = dropped_messages - len(self.failed_changes)
-
-            last_error_set['failed'] = len(self.failed_changes)
-            last_error_set['succeeded'] = len(self.successful_changes)
-            last_error_set['outcome'] = f'MSGS_DROPPED : {dropped_messages}'
-
-        self.status['last_errors_set'] = last_error_set
-        if all_failed:
-            self.context.kafka_status = KafkaStatus.SUBMISSION_FAILURE
-        return
-
-    def clear_changeset(self):
-        self.failed_changes = {}
-        self.successful_changes = []
-        self.change_set = {}
-
-    def get_offset(self):
+    def get_offset(self, sw: SchemaWrapper):
         # Get current offset from Database
-        offset = Offset.get_offset(self.name)
+        offset = Offset.get_offset(sw.schema_id)
         if offset:
-            logger.debug(f'Got offset for {self.name} | {offset}')
+            logger.debug(f'Got offset for {sw.topic} | {offset}')
             return offset
         else:
-            logger.debug(f'Could not get offset for {self.name}, it is a new type')
-            # No valid offset so return None; query will use empty string which is < any value
-            return None
+            logger.debug(f'Could not get offset for {sw.topic}, checking legacy names')
+            return self._migrate_legacy_offset(sw) or None
 
-    def set_offset(self, offset):
+    def set_offset(self, offset, sw: SchemaWrapper):
         # Set a new offset in the database
-        new_offset = Offset.update(self.name, offset)
-        logger.debug(f'Set new offset for {self.name} | {new_offset}')
+        new_offset = Offset.update(sw.schema_id, offset)
+        logger.debug(f'Set new offset for {sw.topic} | {new_offset}')
         self.status['offset'] = new_offset
+
+    # handles move from {AetherName} which can collide over realms -> schema_id which should not
+    # and is what we use to query the entities anyway
+    def _migrate_legacy_offset(self, sw: SchemaWrapper):
+        old_offset = Offset.get_offset(sw.name)
+        if old_offset:
+            logger.warn(f'Found legacy offset for id: {sw.schema_id} at {sw.name}: {old_offset}')
+            self.set_offset(old_offset, sw)
+            logger.warn(f'Migrated offset {sw.name} -> {sw.schema_id}')
+            return old_offset
+        return None
diff --git a/aether-producer/aether/producer/utils.py b/aether-producer/aether/producer/utils.py
new file mode 100644
index 000000000..5df7626bc
--- /dev/null
+++ b/aether-producer/aether/producer/utils.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2019 by eHealth Africa : http://www.eHealthAfrica.org
+#
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+
+
+def halve_iterable(obj):
+    _size = len(obj)
+    _chunk_size = int(_size / 2) + (_size % 2)
+    for i in range(0, _size, _chunk_size):
+        yield obj[i:i + _chunk_size]
+
+
+def utf8size(obj) -> int:
+    if not isinstance(obj, str):
+        try:
+            obj = json.dumps(obj)
+        except json.JSONDecodeError:
+            obj = str(obj)
+    return len(obj.encode('utf-8'))
+
+
+def sanitize_topic(topic):
+    return ''.join(
+        [i if i.isalnum() or i in ['-', '_', '.'] else '_' for i in topic]
+    )
diff --git a/aether-producer/tests/__init__.py b/aether-producer/tests/__init__.py
index a32d143fa..23617fc0d 100644
--- a/aether-producer/tests/__init__.py
+++ b/aether-producer/tests/__init__.py
@@ -60,4 +60,4 @@ def __init__(self):
         self.kafka_status = False
         self.kafka_admin_client = MockAdminInterface()
         self.logger = get_logger('tests')
-        self.topic_managers = {}
+        self.realm_managers = {}
diff --git a/docker-compose-test.yml b/docker-compose-test.yml
index bd18b0962..c9bf54fb5 100644
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@@ -178,7 +178,6 @@ services:
       POSTGRES_HOST: db-test
       POSTGRES_DBNAME: ${TEST_KERNEL_DB_NAME:-test-kernel}
 
-
   # ---------------------------------
   # Aether Integration Tests
   # ---------------------------------
diff --git a/test-aether-integration-module/test/__init__.py b/test-aether-integration-module/test/__init__.py
index c653c2f61..a1d038f75 100644
--- a/test-aether-integration-module/test/__init__.py
+++ b/test-aether-integration-module/test/__init__.py
@@ -27,6 +27,7 @@
 from aether.client.test import (  # noqa
     client,
     project,
+    realm_client,
     schemas,
     schemadecorators,
     mapping,
@@ -80,12 +81,11 @@ def wait_for_producer_status():
             status = producer_request('status')
             if not status:
                 raise ValueError('No status response from producer')
-
             kafka = status.get('kafka_container_accessible')
             if not kafka:
                 raise ValueError('Kafka not connected yet')
 
-            person = status.get('topics', {}).get(KAFKA_SEED_TYPE, {})
+            person = status.get('topics', {}).get(REALM, {}).get(SEED_TYPE, {})
             ok_count = person.get('last_changeset_status', {}).get('succeeded')
             if ok_count:
                 sleep(5)
@@ -110,16 +110,20 @@ def entities(client, schemadecorators):  # noqa: F811
 
 
 @pytest.fixture(scope='function')
-def generate_entities(client, mappingset):  # noqa: F811
-    payloads = iter(fixtures.get_submission_payloads())
-    entities = []
-    for i in range(FORMS_TO_SUBMIT):
-        Submission = client.get_model('Submission')
-        submission = Submission(payload=next(payloads), mappingset=mappingset.id)
-        instance = client.submissions.create(data=submission)
-        for entity in client.entities.paginated('list', submission=instance.id):
-            entities.append(entity)
-    return entities
+def generate_entities(realm_client, mappingset):  # noqa: F811
+
+    def fn(realm):
+        _client = realm_client(realm)
+        payloads = iter(fixtures.get_submission_payloads())
+        entities = []
+        for i in range(FORMS_TO_SUBMIT):
+            Submission = _client.get_model('Submission')
+            submission = Submission(payload=next(payloads), mappingset=mappingset.id)
+            instance = _client.submissions.create(data=submission)
+            for entity in _client.entities.paginated('list', submission=instance.id):
+                entities.append(entity)
+        return entities
+    return fn
 
 
 @pytest.fixture(scope='function')
@@ -146,16 +150,16 @@ def producer_request(endpoint, expect_json=True):
         sleep(1)
 
 
-def topic_status(topic):
+def topic_status(realm, topic):
     status = producer_request('status')
-    return status['topics'][topic]
+    return status['topics'][realm][topic]
 
 
-def producer_topic_count(topic):
+def producer_topic_count(realm, topic):
     status = producer_request('topics')
-    return status[topic]['count']
+    return status[realm][topic]['count']
 
 
-def producer_control_topic(topic, operation):
-    endpoint = f'{operation}?topic={topic}'
+def producer_control_topic(realm, topic, operation):
+    endpoint = f'{operation}?topic={topic}&realm={realm}'
     return producer_request(endpoint, False)
diff --git a/test-aether-integration-module/test/test_integration.py b/test-aether-integration-module/test/test_integration.py
index ae1c9d21b..9a5dab905 100644
--- a/test-aether-integration-module/test/test_integration.py
+++ b/test-aether-integration-module/test/test_integration.py
@@ -34,7 +34,8 @@ def test_1_check_fixtures(project, schemas, schemadecorators, mapping, mappingse
 
 
 def test_2_generate_entities(generate_entities):
-    assert(len(generate_entities) == SEED_ENTITIES)
+    res = generate_entities(REALM)
+    assert(len(res) == SEED_ENTITIES)
 
 
 def test_3_check_updated_count(entities):
@@ -47,8 +48,8 @@ def test_4_check_producer_status(wait_for_producer_status):
 
 
 def test_5_check_producer_topics(producer_topics):
-    assert(KAFKA_SEED_TYPE in producer_topics.keys())
-    assert(int(producer_topics[KAFKA_SEED_TYPE]['count']) == SEED_ENTITIES)
+    assert(REALM in producer_topics.keys())
+    assert(int(producer_topics[REALM][SEED_TYPE]['count']) == SEED_ENTITIES)
 
 
 def test_6_check_stream_entities(read_people, entities):
@@ -61,25 +62,25 @@ def test_6_check_stream_entities(read_people, entities):
 
     assert(len(failed) == 0)
     assert(len(kernel_messages) == len(kafka_messages))
-    assert(producer_topic_count(KAFKA_SEED_TYPE) == len(kafka_messages))
+    assert(producer_topic_count(REALM, SEED_TYPE) == len(kafka_messages))
 
 
 def test_7_control_topic():
-    producer_control_topic(KAFKA_SEED_TYPE, 'pause')
+    producer_control_topic(REALM, SEED_TYPE, 'pause')
     sleep(.5)
 
-    op = topic_status(KAFKA_SEED_TYPE)['operating_status']
+    op = topic_status(REALM, SEED_TYPE)['operating_status']
     assert(op == 'TopicStatus.PAUSED')
-    producer_control_topic(KAFKA_SEED_TYPE, 'resume')
+    producer_control_topic(REALM, SEED_TYPE, 'resume')
     sleep(.5)
 
-    op = topic_status(KAFKA_SEED_TYPE)['operating_status']
+    op = topic_status(REALM, SEED_TYPE)['operating_status']
     assert(op == 'TopicStatus.NORMAL')
-    producer_control_topic(KAFKA_SEED_TYPE, 'rebuild')
+    producer_control_topic(REALM, SEED_TYPE, 'rebuild')
     sleep(.5)
 
     for x in range(120):
-        op = topic_status(KAFKA_SEED_TYPE)['operating_status']
+        op = topic_status(REALM, SEED_TYPE)['operating_status']
         if op != 'TopicStatus.REBUILDING':
             return
         sleep(1)

From dda3bed6b1af432f3a2907dea135ee78793ccabb Mon Sep 17 00:00:00 2001
From: shawnsarwar <shawn.sarwar@ehealthnigeria.org>
Date: Tue, 7 Jul 2020 10:40:42 +0200
Subject: [PATCH 23/29] fix: the presence of artifacts without realms were
 causing an issue when connecting via database (#870)

---
 aether-producer/aether/producer/kernel_db.py | 2 +-
 aether-producer/aether/producer/topic.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/aether-producer/aether/producer/kernel_db.py b/aether-producer/aether/producer/kernel_db.py
index bc767b7ea..af6725d3b 100644
--- a/aether-producer/aether/producer/kernel_db.py
+++ b/aether-producer/aether/producer/kernel_db.py
@@ -124,7 +124,7 @@ def mode(self):
     def get_realms(self):
         query = sql.SQL(_REALMS_SQL)
         cursor = self._exec_sql('get_realms', 1, query)
-        return [row['realm'] for row in cursor]
+        return [row['realm'] for row in cursor if row['realm']]
 
     def get_schemas(self, realm=None):
         self.last_check = datetime.now().isoformat()
diff --git a/aether-producer/aether/producer/topic.py b/aether-producer/aether/producer/topic.py
index a2c9fc37a..cfddf69b0 100644
--- a/aether-producer/aether/producer/topic.py
+++ b/aether-producer/aether/producer/topic.py
@@ -36,7 +36,7 @@
 from aether.producer.db import Offset
 from aether.producer.settings import SETTINGS, KAFKA_SETTINGS, get_logger
 
-logger = get_logger('producer-topic')
+logger = get_logger('topic')
 
 
 class SchemaWrapper(object):

From bbc417f34c324e687c3a9e8f350fa13d496c7990 Mon Sep 17 00:00:00 2001
From: shawnsarwar <shawn.sarwar@ehealthnigeria.org>
Date: Tue, 7 Jul 2020 11:13:39 +0200
Subject: [PATCH 24/29] fix: empty realm (#871)

* fix: apply filter to api realms to check for empty
---
 aether-producer/aether/producer/kernel_api.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/aether-producer/aether/producer/kernel_api.py b/aether-producer/aether/producer/kernel_api.py
index 57262382e..58ffe78f1 100644
--- a/aether-producer/aether/producer/kernel_api.py
+++ b/aether-producer/aether/producer/kernel_api.py
@@ -69,7 +69,10 @@ def mode(self):
         return 'api'
 
     def get_realms(self):
-        return self._fetch(url=_REALMS_URL)['realms']
+        return [
+            r for r in self._fetch(url=_REALMS_URL)['realms']
+            if r  # realm "" can exist, so we must filter for it.
+        ]
 
     def get_schemas(self, realm=None):
         self.last_check = datetime.now().isoformat()

From 2f47fe41430b541d0de10dff830c835d5196d82c Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 20 Jul 2020 12:55:46 +0200
Subject: [PATCH 25/29] chore: upgrade dependencies (#874)

---
 aether-kernel/conf/pip/requirements.txt       | 30 +++++++++----------
 aether-odk-module/conf/pip/requirements.txt   | 24 +++++++--------
 aether-producer/conf/pip/requirements.txt     |  6 ++--
 .../aether/ui/assets/css/base/_fonts.scss     |  4 +--
 aether-ui/aether/ui/assets/package.json       |  8 ++---
 aether-ui/conf/pip/requirements.txt           | 22 +++++++-------
 6 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 0fe11af6f..5b71dc9f3 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -12,30 +12,30 @@
 #
 ################################################################################
 
-aether.python==1.0.17
+aether.python==1.1.0
 aether.sdk==1.3.1
 attrs==19.3.0
 autopep8==1.5.3
-boto3==1.14.9
-botocore==1.17.9
-cachetools==4.1.0
+boto3==1.14.23
+botocore==1.17.23
+cachetools==4.1.1
 certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
 coreapi==2.3.3
 coreschema==0.0.4
-coverage==5.1
+coverage==5.2
 cryptography==2.9.2
 decorator==4.4.2
-Django==2.2.13
-django-cacheops==5.0
+Django==2.2.14
+django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
 django-debug-toolbar==2.2
 django-dynamic-fixture==3.1.0
 django-filter==2.3.0
-django-minio-storage==0.3.7
+django-minio-storage==0.3.8
 django-model-utils==4.0.0
 django-postgrespool2==1.0.1
 django-prometheus==2.0.0
@@ -53,14 +53,13 @@ flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
 google-api-core==1.21.0
-google-auth==1.18.0
+google-auth==1.19.2
 google-cloud-core==1.3.0
 google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
-idna==2.9
-importlib-metadata==1.6.1
+idna==2.10
 inflection==0.5.0
 itypes==1.2.0
 jdcal==1.4.1
@@ -68,11 +67,11 @@ Jinja2==2.11.2
 jmespath==0.10.0
 jsonpath-ng==1.5.1
 jsonschema==3.2.0
-lxml==4.5.1
+lxml==4.5.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
 minio==5.0.10
-openpyxl==3.0.3
+openpyxl==3.0.4
 packaging==20.4
 ply==3.11
 prometheus-client==0.8.0
@@ -96,14 +95,13 @@ rsa==4.6
 ruamel.yaml==0.16.10
 ruamel.yaml.clib==0.2.0
 s3transfer==0.3.3
-sentry-sdk==0.15.1
+sentry-sdk==0.16.1
 six==1.15.0
 spavro==1.1.23
-SQLAlchemy==1.3.17
+SQLAlchemy==1.3.18
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1
 uritemplate==3.0.1
 urllib3==1.25.9
 uWSGI==2.0.19.1
-zipp==3.1.0
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index 2fbea0af3..f7cbfbbaa 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -14,21 +14,21 @@
 
 aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.14.9
-botocore==1.17.9
-cachetools==4.1.0
+boto3==1.14.23
+botocore==1.17.23
+cachetools==4.1.1
 certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
-coverage==5.1
+coverage==5.2
 cryptography==2.9.2
-Django==2.2.13
-django-cacheops==5.0
+Django==2.2.14
+django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
 django-debug-toolbar==2.2
-django-minio-storage==0.3.7
+django-minio-storage==0.3.8
 django-postgrespool2==1.0.1
 django-prometheus==2.0.0
 django-redis==4.12.1
@@ -43,17 +43,17 @@ flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
 google-api-core==1.21.0
-google-auth==1.18.0
+google-auth==1.19.2
 google-cloud-core==1.3.0
 google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
-idna==2.9
+idna==2.10
 Jinja2==2.11.2
 jmespath==0.10.0
 linecache2==1.0.0
-lxml==4.5.1
+lxml==4.5.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
 minio==5.0.10
@@ -75,10 +75,10 @@ redis==3.5.3
 requests==2.24.0
 rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.15.1
+sentry-sdk==0.16.1
 six==1.15.0
 spavro==1.1.23
-SQLAlchemy==1.3.17
+SQLAlchemy==1.3.18
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index f55b41408..426a0f154 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -24,7 +24,7 @@ flake8-quotes==3.2.0
 Flask==1.1.2
 gevent==20.6.2
 greenlet==0.4.16
-idna==2.9
+idna==2.10
 itsdangerous==1.1.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
@@ -34,7 +34,7 @@ packaging==20.4
 pluggy==0.13.1
 psycogreen==1.0.2
 psycopg2-binary==2.8.5
-py==1.8.2
+py==1.9.0
 pycodestyle==2.6.0
 pycparser==2.20
 pyflakes==2.2.0
@@ -44,7 +44,7 @@ pytest==5.4.3
 requests==2.24.0
 six==1.15.0
 spavro==1.1.23
-SQLAlchemy==1.3.17
+SQLAlchemy==1.3.18
 urllib3==1.25.9
 wcwidth==0.2.5
 Werkzeug==1.0.1
diff --git a/aether-ui/aether/ui/assets/css/base/_fonts.scss b/aether-ui/aether/ui/assets/css/base/_fonts.scss
index 91ec9a224..123ed36bd 100644
--- a/aether-ui/aether/ui/assets/css/base/_fonts.scss
+++ b/aether-ui/aether/ui/assets/css/base/_fonts.scss
@@ -23,5 +23,5 @@
 @import url('https://fonts.googleapis.com/css?family=Fira+Mono');
 
 /* Font Awesome 5  https://fontawesome.com/ */
-@import url('https://use.fontawesome.com/releases/v5.13.0/css/fontawesome.css');
-@import url('https://use.fontawesome.com/releases/v5.13.0/css/solid.css');
+@import url('https://use.fontawesome.com/releases/v5.14.0/css/fontawesome.css');
+@import url('https://use.fontawesome.com/releases/v5.14.0/css/solid.css');
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index e4faba27d..4b6c4179d 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -29,7 +29,7 @@
     "react": "~16.13.0",
     "react-clipboard.js": "~2.0.16",
     "react-dom": "~16.13.0",
-    "react-intl": "~4.7.0",
+    "react-intl": "~5.3.0",
     "react-outside-click-handler": "~1.3.0",
     "react-redux": "~7.2.0",
     "react-router-dom": "~5.2.0",
@@ -38,7 +38,7 @@
     "uuid": "~8.2.0",
     "webpack-google-cloud-storage-plugin": "~0.9.0",
     "webpack-s3-plugin": "~1.0.3",
-    "whatwg-fetch": "~3.0.0"
+    "whatwg-fetch": "~3.2.0"
   },
   "devDependencies": {
     "@babel/core": "~7.10.0",
@@ -48,7 +48,7 @@
     "@hot-loader/react-dom": "~16.13.0",
     "babel-loader": "~8.1.0",
     "css-loader": "~3.6.0",
-    "eslint": "~7.3.0",
+    "eslint": "~7.5.0",
     "enzyme": "~3.11.0",
     "enzyme-adapter-react-16": "~1.15.0",
     "express": "~4.17.0",
@@ -59,7 +59,7 @@
     "node-sass": "~4.14.0",
     "react-hot-loader": "~4.12.0",
     "redux-devtools-extension": "~2.13.0",
-    "sass-loader": "~8.0.0",
+    "sass-loader": "~9.0.0",
     "standard": "~14.3.0",
     "style-loader": "~1.2.0",
     "stylelint": "~13.6.0",
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index a7e831cf3..128b3aba2 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -14,21 +14,21 @@
 
 aether.sdk==1.3.1
 autopep8==1.5.3
-boto3==1.14.9
-botocore==1.17.9
-cachetools==4.1.0
+boto3==1.14.23
+botocore==1.17.23
+cachetools==4.1.1
 certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 configparser==5.0.0
-coverage==5.1
+coverage==5.2
 cryptography==2.9.2
-Django==2.2.13
-django-cacheops==5.0
+Django==2.2.14
+django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
 django-debug-toolbar==2.2
-django-minio-storage==0.3.7
+django-minio-storage==0.3.8
 django-model-utils==4.0.0
 django-postgrespool2==1.0.1
 django-prometheus==2.0.0
@@ -44,13 +44,13 @@ flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
 google-api-core==1.21.0
-google-auth==1.18.0
+google-auth==1.19.2
 google-cloud-core==1.3.0
 google-cloud-storage==1.29.0
 google-resumable-media==0.5.1
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
-idna==2.9
+idna==2.10
 Jinja2==2.11.2
 jmespath==0.10.0
 MarkupSafe==1.1.1
@@ -73,9 +73,9 @@ redis==3.5.3
 requests==2.24.0
 rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.15.1
+sentry-sdk==0.16.1
 six==1.15.0
-SQLAlchemy==1.3.17
+SQLAlchemy==1.3.18
 sqlparse==0.3.1
 tblib==1.6.0
 toml==0.10.1

From d9bbb746ad8535eacddd2538d00ace8e41dcf9eb Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Tue, 21 Jul 2020 09:48:57 +0200
Subject: [PATCH 26/29] fix(producer): remove "callable" warnings (#876)

---
 aether-producer/aether/producer/__init__.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/aether-producer/aether/producer/__init__.py b/aether-producer/aether/producer/__init__.py
index 1170e1c90..cdc85d999 100644
--- a/aether-producer/aether/producer/__init__.py
+++ b/aether-producer/aether/producer/__init__.py
@@ -130,21 +130,25 @@ def broker_info(self):
                     res['brokers'].append(f'{b}')
 
             for t in iter(md.topics.values()):
-                t_str = []
-                t_str.append(
+                topics = []
+
+                msg_t = (
                     f'{t} with {len(t.partitions)} partition(s)'
                     (f', error: {t.error}' if t.error is not None else '')
                 )
+                topics.append(msg_t)
 
                 for p in iter(t.partitions.values()):
-                    t_str.append(
+                    msg_p = (
                         f'partition {p.id}'
                         f', leader: {p.leader}'
                         f', replicas: {p.replicas}'
                         f', isrs: {p.isrs}'
                         (f', error: {p.error}' if p.error is not None else '')
                     )
-                res['topics'].append(t_str)
+                    topics.append(msg_p)
+
+                res['topics'].append(topics)
             return res
         except Exception as err:
             return {'error': f'{err}'}

From 83650a8455b62634de28fb7039d23c6fbb6cbef7 Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Wed, 5 Aug 2020 13:00:22 +0200
Subject: [PATCH 27/29] chore: upgrade dependencies (#878)

---
 aether-kernel/conf/pip/requirements.txt     | 39 +++++++++++----------
 aether-odk-module/conf/pip/requirements.txt | 37 +++++++++----------
 aether-producer/conf/pip/requirements.txt   | 13 +++----
 aether-ui/aether/ui/assets/package.json     | 16 ++++-----
 aether-ui/conf/pip/requirements.txt         | 37 +++++++++----------
 5 files changed, 73 insertions(+), 69 deletions(-)

diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 5b71dc9f3..3e2cee6df 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -12,23 +12,23 @@
 #
 ################################################################################
 
-aether.python==1.1.0
+aether.python==1.1.2
 aether.sdk==1.3.1
 attrs==19.3.0
-autopep8==1.5.3
-boto3==1.14.23
-botocore==1.17.23
+autopep8==1.5.4
+boto3==1.14.35
+botocore==1.17.35
 cachetools==4.1.1
 certifi==2020.6.20
-cffi==1.14.0
+cffi==1.14.1
 chardet==3.0.4
 configparser==5.0.0
 coreapi==2.3.3
 coreschema==0.0.4
-coverage==5.2
-cryptography==2.9.2
+coverage==5.2.1
+cryptography==3.0
 decorator==4.4.2
-Django==2.2.14
+Django==2.2.15
 django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
@@ -43,7 +43,7 @@ django-redis==4.12.1
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
-djangorestframework==3.11.0
+djangorestframework==3.11.1
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
 drf-yasg==1.17.1
@@ -52,11 +52,12 @@ et-xmlfile==1.0.1
 flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.21.0
-google-auth==1.19.2
-google-cloud-core==1.3.0
-google-cloud-storage==1.29.0
-google-resumable-media==0.5.1
+google-api-core==1.22.0
+google-auth==1.20.0
+google-cloud-core==1.4.0
+google-cloud-storage==1.30.0
+google-crc32c==0.1.0
+google-resumable-media==0.7.0
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.10
@@ -70,12 +71,12 @@ jsonschema==3.2.0
 lxml==4.5.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
-minio==5.0.10
+minio==6.0.0
 openpyxl==3.0.4
 packaging==20.4
 ply==3.11
 prometheus-client==0.8.0
-protobuf==3.12.2
+protobuf==3.12.4
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -95,13 +96,13 @@ rsa==4.6
 ruamel.yaml==0.16.10
 ruamel.yaml.clib==0.2.0
 s3transfer==0.3.3
-sentry-sdk==0.16.1
+sentry-sdk==0.16.3
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.18
 sqlparse==0.3.1
-tblib==1.6.0
+tblib==1.7.0
 toml==0.10.1
 uritemplate==3.0.1
-urllib3==1.25.9
+urllib3==1.25.10
 uWSGI==2.0.19.1
diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt
index f7cbfbbaa..178eabbad 100644
--- a/aether-odk-module/conf/pip/requirements.txt
+++ b/aether-odk-module/conf/pip/requirements.txt
@@ -13,17 +13,17 @@
 ################################################################################
 
 aether.sdk==1.3.1
-autopep8==1.5.3
-boto3==1.14.23
-botocore==1.17.23
+autopep8==1.5.4
+boto3==1.14.35
+botocore==1.17.35
 cachetools==4.1.1
 certifi==2020.6.20
-cffi==1.14.0
+cffi==1.14.1
 chardet==3.0.4
 configparser==5.0.0
-coverage==5.2
-cryptography==2.9.2
-Django==2.2.14
+coverage==5.2.1
+cryptography==3.0
+Django==2.2.15
 django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
@@ -35,18 +35,19 @@ django-redis==4.12.1
 django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
-djangorestframework==3.11.0
+djangorestframework==3.11.1
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
 flake8==3.8.3
 flake8-quotes==3.2.0
 FormEncode==1.3.1
 funcy==1.14
-google-api-core==1.21.0
-google-auth==1.19.2
-google-cloud-core==1.3.0
-google-cloud-storage==1.29.0
-google-resumable-media==0.5.1
+google-api-core==1.22.0
+google-auth==1.20.0
+google-cloud-core==1.4.0
+google-cloud-storage==1.30.0
+google-crc32c==0.1.0
+google-resumable-media==0.7.0
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.10
@@ -56,9 +57,9 @@ linecache2==1.0.0
 lxml==4.5.2
 MarkupSafe==1.1.1
 mccabe==0.6.1
-minio==5.0.10
+minio==6.0.0
 prometheus-client==0.8.0
-protobuf==3.12.2
+protobuf==3.12.4
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -75,16 +76,16 @@ redis==3.5.3
 requests==2.24.0
 rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.16.1
+sentry-sdk==0.16.3
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.18
 sqlparse==0.3.1
-tblib==1.6.0
+tblib==1.7.0
 toml==0.10.1
 traceback2==1.4.0
 unicodecsv==0.14.1
 unittest2==1.1.0
-urllib3==1.25.9
+urllib3==1.25.10
 uWSGI==2.0.19.1
 xlrd==1.2.0
diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
index 426a0f154..c472a3dcc 100644
--- a/aether-producer/conf/pip/requirements.txt
+++ b/aether-producer/conf/pip/requirements.txt
@@ -14,17 +14,18 @@
 
 attrs==19.3.0
 certifi==2020.6.20
-cffi==1.14.0
+cffi==1.14.1
 chardet==3.0.4
 click==7.1.2
-confluent-kafka==1.4.2
-cryptography==2.9.2
+confluent-kafka==1.5.0
+cryptography==3.0
 flake8==3.8.3
 flake8-quotes==3.2.0
 Flask==1.1.2
 gevent==20.6.2
 greenlet==0.4.16
 idna==2.10
+iniconfig==1.0.1
 itsdangerous==1.1.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
@@ -40,13 +41,13 @@ pycparser==2.20
 pyflakes==2.2.0
 pyOpenSSL==19.1.0
 pyparsing==2.4.7
-pytest==5.4.3
+pytest==6.0.1
 requests==2.24.0
 six==1.15.0
 spavro==1.1.23
 SQLAlchemy==1.3.18
-urllib3==1.25.9
-wcwidth==0.2.5
+toml==0.10.1
+urllib3==1.25.10
 Werkzeug==1.0.1
 zope.event==4.4
 zope.interface==5.1.0
diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json
index 4b6c4179d..405b361fc 100644
--- a/aether-ui/aether/ui/assets/package.json
+++ b/aether-ui/aether/ui/assets/package.json
@@ -29,26 +29,26 @@
     "react": "~16.13.0",
     "react-clipboard.js": "~2.0.16",
     "react-dom": "~16.13.0",
-    "react-intl": "~5.3.0",
+    "react-intl": "~5.4.0",
     "react-outside-click-handler": "~1.3.0",
     "react-redux": "~7.2.0",
     "react-router-dom": "~5.2.0",
     "redux": "~4.0.0",
     "redux-thunk": "~2.3.0",
-    "uuid": "~8.2.0",
+    "uuid": "~8.3.0",
     "webpack-google-cloud-storage-plugin": "~0.9.0",
     "webpack-s3-plugin": "~1.0.3",
-    "whatwg-fetch": "~3.2.0"
+    "whatwg-fetch": "~3.3.0"
   },
   "devDependencies": {
-    "@babel/core": "~7.10.0",
+    "@babel/core": "~7.11.0",
     "@babel/plugin-proposal-class-properties": "~7.10.0",
-    "@babel/preset-env": "~7.10.0",
+    "@babel/preset-env": "~7.11.0",
     "@babel/preset-react": "~7.10.0",
     "@hot-loader/react-dom": "~16.13.0",
     "babel-loader": "~8.1.0",
-    "css-loader": "~3.6.0",
-    "eslint": "~7.5.0",
+    "css-loader": "~4.2.0",
+    "eslint": "~7.6.0",
     "enzyme": "~3.11.0",
     "enzyme-adapter-react-16": "~1.15.0",
     "express": "~4.17.0",
@@ -64,7 +64,7 @@
     "style-loader": "~1.2.0",
     "stylelint": "~13.6.0",
     "stylelint-config-standard": "~20.0.0",
-    "webpack": "~4.43.0",
+    "webpack": "~4.44.0",
     "webpack-bundle-tracker": "~0.4.3",
     "webpack-cli": "~3.3.0",
     "webpack-dev-middleware": "~3.7.0",
diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt
index 128b3aba2..06d672867 100644
--- a/aether-ui/conf/pip/requirements.txt
+++ b/aether-ui/conf/pip/requirements.txt
@@ -13,17 +13,17 @@
 ################################################################################
 
 aether.sdk==1.3.1
-autopep8==1.5.3
-boto3==1.14.23
-botocore==1.17.23
+autopep8==1.5.4
+boto3==1.14.35
+botocore==1.17.35
 cachetools==4.1.1
 certifi==2020.6.20
-cffi==1.14.0
+cffi==1.14.1
 chardet==3.0.4
 configparser==5.0.0
-coverage==5.2
-cryptography==2.9.2
-Django==2.2.14
+coverage==5.2.1
+cryptography==3.0
+Django==2.2.15
 django-cacheops==5.0.1
 django-cleanup==5.0.0
 django-cors-headers==3.4.0
@@ -37,17 +37,18 @@ django-silk==4.0.1
 django-storages==1.9.1
 django-uwsgi==0.2.2
 django-webpack-loader==0.7.0
-djangorestframework==3.11.0
+djangorestframework==3.11.1
 docutils==0.15.2
 drf-dynamic-fields==0.3.1
 flake8==3.8.3
 flake8-quotes==3.2.0
 funcy==1.14
-google-api-core==1.21.0
-google-auth==1.19.2
-google-cloud-core==1.3.0
-google-cloud-storage==1.29.0
-google-resumable-media==0.5.1
+google-api-core==1.22.0
+google-auth==1.20.0
+google-cloud-core==1.4.0
+google-cloud-storage==1.30.0
+google-crc32c==0.1.0
+google-resumable-media==0.7.0
 googleapis-common-protos==1.52.0
 gprof2dot==2019.11.30
 idna==2.10
@@ -55,9 +56,9 @@ Jinja2==2.11.2
 jmespath==0.10.0
 MarkupSafe==1.1.1
 mccabe==0.6.1
-minio==5.0.10
+minio==6.0.0
 prometheus-client==0.8.0
-protobuf==3.12.2
+protobuf==3.12.4
 psycopg2-binary==2.8.5
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -73,11 +74,11 @@ redis==3.5.3
 requests==2.24.0
 rsa==4.6
 s3transfer==0.3.3
-sentry-sdk==0.16.1
+sentry-sdk==0.16.3
 six==1.15.0
 SQLAlchemy==1.3.18
 sqlparse==0.3.1
-tblib==1.6.0
+tblib==1.7.0
 toml==0.10.1
-urllib3==1.25.9
+urllib3==1.25.10
 uWSGI==2.0.19.1

From 86314e16ec973aa38c4260d5c34febf4bd5f3811 Mon Sep 17 00:00:00 2001
From: obdulia <obdulia.losantos@gmail.com>
Date: Fri, 7 Aug 2020 13:57:35 +0200
Subject: [PATCH 28/29] docs: python 3.8 [ci-skip]

---
 README.md               | 2 +-
 docker-compose-base.yml | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 487fe0827..24a792c45 100644
--- a/README.md
+++ b/README.md
@@ -149,7 +149,7 @@ Also check the aether sdk section about [environment variables](https://github.c
 - `LOGGING_FORMATTER`: `json`. The app messages format.
   Possible values: `verbose` or `json`.
 - `LOGGING_LEVEL`: `info` Logging level for app messages.
-  https://docs.python.org/3.7/library/logging.html#levels
+  https://docs.python.org/3.8/library/logging.html#levels
 
 - `DEBUG` Enables debug mode. Is `false` if unset or set to empty string,
   anything else is considered `true`.
diff --git a/docker-compose-base.yml b/docker-compose-base.yml
index 13ad3a756..5903437fe 100644
--- a/docker-compose-base.yml
+++ b/docker-compose-base.yml
@@ -157,9 +157,9 @@ services:
       - ./aether-kernel:/code
       # -------------------------------------------------------------
       # to speed up SDK development changes
-      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk
+      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk
       # to speed up Aether Python Library development changes
-      # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/usr/local/lib/python3.7/site-packages/aether/python
+      # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/usr/local/lib/python3.8/site-packages/aether/python
     command: start_dev
 
 
@@ -232,7 +232,7 @@ services:
       - ./aether-odk-module:/code
       # -------------------------------------------------------------
       # to speed up SDK development changes
-      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk
+      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk
     command: start_dev
 
 
@@ -297,7 +297,7 @@ services:
       - ./aether-ui:/code
       # -------------------------------------------------------------
       # to speed up SDK development changes
-      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk
+      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk
     command: start_dev
 
   ui-assets-base:

From 3516d97b0829e872eaa0cf7d089f0671bf63daba Mon Sep 17 00:00:00 2001
From: Obdulia Losantos <obdulia.losantos@ehealthnigeria.org>
Date: Mon, 10 Aug 2020 17:37:41 +0200
Subject: [PATCH 29/29] fix(extractor): generate unique ids per mapping (#879)

* fix(extractor): generate unique ids per mapping

* fix: clean utils module tests

* fix: cleaning
---
 .../aether/kernel/api/entity_extractor.py     | 33 ++++---
 .../aether/kernel/api/serializers.py          |  4 +-
 .../aether/kernel/api/tests/test_exporter.py  | 12 ++-
 .../aether/kernel/api/tests/test_filters.py   | 15 ++-
 .../aether/kernel/api/tests/test_models.py    |  4 +-
 .../kernel/api/tests/test_serializers.py      | 15 +--
 .../aether/kernel/api/tests/test_utils.py     | 98 +++++++------------
 .../aether/kernel/api/tests/test_views.py     | 38 +++++--
 aether-kernel/aether/kernel/api/views.py      | 25 +++--
 .../management/commands/extract_entities.py   |  2 +-
 .../aether/kernel/tests/test_commands.py      | 22 ++---
 aether-kernel/conf/pip/requirements.txt       |  2 +-
 12 files changed, 135 insertions(+), 135 deletions(-)

diff --git a/aether-kernel/aether/kernel/api/entity_extractor.py b/aether-kernel/aether/kernel/api/entity_extractor.py
index 27b1329db..d0aeb861f 100644
--- a/aether-kernel/aether/kernel/api/entity_extractor.py
+++ b/aether-kernel/aether/kernel/api/entity_extractor.py
@@ -18,7 +18,10 @@
 
 from django.db import transaction
 
-from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, extract_create_entities
+from aether.python.entity.extractor import (
+    ENTITY_EXTRACTION_ERRORS as KEY,
+    extract_create_entities,
+)
 from . import models
 
 
@@ -30,7 +33,7 @@ def run_entity_extraction(submission, overwrite=False):
         # replace their payloads with the new ones
         submission.entities.all().delete()
         payload = submission.payload
-        del payload[ENTITY_EXTRACTION_ERRORS]
+        payload.pop(KEY, None)
         submission.payload = payload
         submission.is_extracted = False
         submission.save(update_fields=['payload', 'is_extracted'])
@@ -43,35 +46,36 @@ def run_entity_extraction(submission, overwrite=False):
                          .exclude(definition__entities__isnull=True) \
                          .exclude(definition__entities={})
 
+    payload = dict(submission.payload)
     for mapping in mappings:
         # Get the primary key of the schemadecorator
         entity_sd_ids = mapping.definition.get('entities')
         # Get the schema of the schemadecorator
-        schema_decorator = {
+        schema_decorators = {
             name: models.SchemaDecorator.objects.get(pk=_id)
             for name, _id in entity_sd_ids.items()
         }
         schemas = {
-            name: ps.schema.definition
-            for name, ps in schema_decorator.items()
+            name: sd.schema.definition
+            for name, sd in schema_decorators.items()
         }
-        _, entities = extract_create_entities(
-            submission_payload=submission.payload,
+        payload, entities = extract_create_entities(
+            submission_payload=payload,
             mapping_definition=mapping.definition,
             schemas=schemas,
+            mapping_id=mapping.id,
         )
         for entity in entities:
-            schemadecorator_name = entity.schemadecorator_name
-            schemadecorator = schema_decorator[schemadecorator_name]
-            entity_instance = models.Entity(
+            models.Entity(
+                id=entity.id,
                 payload=entity.payload,
                 status=entity.status,
-                schemadecorator=schemadecorator,
+                schemadecorator=schema_decorators[entity.schemadecorator_name],
                 submission=submission,
                 mapping=mapping,
-                mapping_revision=mapping.revision
-            )
-            entity_instance.save()
+                mapping_revision=mapping.revision,
+                project=submission.project,
+            ).save()
 
     # this should include in the submission payload the following properties
     # generated during the extraction:
@@ -79,5 +83,6 @@ def run_entity_extraction(submission, overwrite=False):
     #   to create the entities.
     # - ``aether_extractor_enrichment``, with the generated values that allow us
     #   to re-execute this process again with the same result.
+    submission.payload = payload
     submission.is_extracted = submission.entities.count() > 0
     submission.save(update_fields=['payload', 'is_extracted'])
diff --git a/aether-kernel/aether/kernel/api/serializers.py b/aether-kernel/aether/kernel/api/serializers.py
index d59772f07..36b8cc84c 100644
--- a/aether-kernel/aether/kernel/api/serializers.py
+++ b/aether-kernel/aether/kernel/api/serializers.py
@@ -35,8 +35,8 @@
 from aether.python import utils
 from aether.python.constants import MergeOptions as MERGE_OPTIONS
 from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
-from .entity_extractor import run_entity_extraction
 
+from .entity_extractor import run_entity_extraction
 from . import models, validators
 
 
@@ -204,7 +204,7 @@ def create(self, validated_data):
         instance = super(SubmissionSerializer, self).create(validated_data)
         try:
             run_entity_extraction(instance)
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             instance.payload[ENTITY_EXTRACTION_ERRORS] = instance.payload.get(ENTITY_EXTRACTION_ERRORS, [])
             instance.payload[ENTITY_EXTRACTION_ERRORS] += [str(e)]
             instance.save()
diff --git a/aether-kernel/aether/kernel/api/tests/test_exporter.py b/aether-kernel/aether/kernel/api/tests/test_exporter.py
index ba6faea73..891095443 100644
--- a/aether-kernel/aether/kernel/api/tests/test_exporter.py
+++ b/aether-kernel/aether/kernel/api/tests/test_exporter.py
@@ -383,7 +383,7 @@ def setUp(self):
             EXAMPLE_SCHEMA = json.load(infile)
 
         with open(os.path.join(here, 'files/export.json'), 'rb') as infile:
-            EXAMPLE_PAYLOAD = json.load(infile)
+            self.EXAMPLE_PAYLOAD = json.load(infile)
 
         project = models.Project.objects.create(
             name='project1',
@@ -400,7 +400,7 @@ def setUp(self):
             }],
         )
         submission = models.Submission.objects.create(
-            payload=EXAMPLE_PAYLOAD,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=models.MappingSet.objects.get(pk=artefacts_id),
         )
         # extract entities
@@ -714,11 +714,11 @@ def test__generate__xlsx__paginate(self):
 
         submission_1 = models.Submission.objects.first()
         submission_2 = models.Submission.objects.create(
-            payload=submission_1.payload,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=submission_1.mappingset,
         )
         submission_3 = models.Submission.objects.create(
-            payload=submission_1.payload,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=submission_1.mappingset,
         )
 
@@ -841,7 +841,7 @@ def test_submissions_export__xlsx__error(self, *args):
     def test_submissions_export__csv__error(self, *args):
         for i in range(13):
             models.Submission.objects.create(
-                payload={'name': f'Person-{i}'},
+                payload=dict({'name': f'Person-{i}'}),
                 mappingset=models.MappingSet.objects.first(),
             )
 
@@ -1225,6 +1225,7 @@ def test_entities_export__attachments(self):
 
         # new submission with 2 attachments
         submission.pk = None
+        submission.payload = dict(self.EXAMPLE_PAYLOAD)
         submission.save()
         self.assertEqual(models.Submission.objects.count(), 2)
 
@@ -1244,6 +1245,7 @@ def test_entities_export__attachments(self):
 
         # new submission without attachments
         submission.pk = None
+        submission.payload = dict(self.EXAMPLE_PAYLOAD)
         submission.save()
         self.assertEqual(models.Submission.objects.count(), 3)
         run_entity_extraction(submission)
diff --git a/aether-kernel/aether/kernel/api/tests/test_filters.py b/aether-kernel/aether/kernel/api/tests/test_filters.py
index 8b46d335a..0080f32e1 100644
--- a/aether-kernel/aether/kernel/api/tests/test_filters.py
+++ b/aether-kernel/aether/kernel/api/tests/test_filters.py
@@ -23,6 +23,8 @@
 from django.test import TestCase, override_settings
 from django.urls import reverse
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, ENTITY_EXTRACTION_ENRICHMENT
+
 from aether.kernel.api import models
 from aether.kernel.api.filters import EntityFilter, SubmissionFilter
 from aether.kernel.api.tests.utils.generators import (
@@ -30,6 +32,11 @@
     generate_random_string,
 )
 
+ENTITY_EXTRACTION_FIELDS = [
+    ENTITY_EXTRACTION_ERRORS,
+    ENTITY_EXTRACTION_ENRICHMENT,
+]
+
 
 @override_settings(MULTITENANCY=False)
 class TestFilters(TestCase):
@@ -570,12 +577,12 @@ def test_submission_filter__by_payload(self):
                 submission_payload = {
                     k: v
                     for k, v in submission['payload'].items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 original_payload = {
                     k: v
                     for k, v in payload.items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 self.assertEqual(submission_payload, original_payload)
         self.assertEqual(submissions_count, filtered_submissions_count)
@@ -606,12 +613,12 @@ def test_submission_filter__by_payload__post(self):
                 submission_payload = {
                     k: v
                     for k, v in submission['payload'].items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 original_payload = {
                     k: v
                     for k, v in payload.items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 self.assertEqual(submission_payload, original_payload)
         self.assertEqual(submissions_count, filtered_submissions_count)
diff --git a/aether-kernel/aether/kernel/api/tests/test_models.py b/aether-kernel/aether/kernel/api/tests/test_models.py
index a08f73a65..88b85913a 100644
--- a/aether-kernel/aether/kernel/api/tests/test_models.py
+++ b/aether-kernel/aether/kernel/api/tests/test_models.py
@@ -142,7 +142,7 @@ def test_models(self):
 
         submission = models.Submission.objects.create(
             revision='a sample revision',
-            payload=EXAMPLE_SOURCE_DATA,
+            payload=dict(EXAMPLE_SOURCE_DATA),
             mappingset=mappingset,
         )
         self.assertNotEqual(models.Submission.objects.count(), 0)
@@ -187,7 +187,7 @@ def test_models(self):
         with self.assertRaises(IntegrityError) as err4:
             models.Entity.objects.create(
                 revision='a sample revision',
-                payload=EXAMPLE_SOURCE_DATA,  # this is the submission payload without ID
+                payload=dict(EXAMPLE_SOURCE_DATA),  # this is the submission payload without ID
                 status='Publishable',
                 schemadecorator=schemadecorator,
             )
diff --git a/aether-kernel/aether/kernel/api/tests/test_serializers.py b/aether-kernel/aether/kernel/api/tests/test_serializers.py
index 845059654..51e89a536 100644
--- a/aether-kernel/aether/kernel/api/tests/test_serializers.py
+++ b/aether-kernel/aether/kernel/api/tests/test_serializers.py
@@ -22,6 +22,8 @@
 from django.test import RequestFactory, TestCase
 from rest_framework.serializers import ValidationError
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ENRICHMENT
+
 from aether.kernel.api import models, serializers
 
 from . import EXAMPLE_SCHEMA, EXAMPLE_SOURCE_DATA, EXAMPLE_SOURCE_DATA_ENTITY, EXAMPLE_MAPPING
@@ -174,7 +176,7 @@ def test__serializers__create_and_update(self):
         submission = serializers.SubmissionSerializer(
             data={
                 'project': project.data['id'],
-                'payload': EXAMPLE_SOURCE_DATA,
+                'payload': dict(EXAMPLE_SOURCE_DATA),
             },
             context={'request': self.request},
         )
@@ -197,18 +199,17 @@ def test__serializers__create_and_update(self):
         self.assertTrue(submission.is_valid(), submission.errors)
 
         # save the submission and check that no entities were created
-        # and we have aether errors
         self.assertEqual(models.Entity.objects.count(), 0)
         submission.save()
         self.assertEqual(models.Entity.objects.count(), 0)
-        self.assertIn('aether_errors', submission.data['payload'])
+        self.assertFalse(submission.data['is_extracted'])
 
         # check the submission without entity extraction errors
         submission = serializers.SubmissionSerializer(
             data={
                 'mappingset': mappingset.data['id'],
                 'project': project.data['id'],
-                'payload': EXAMPLE_SOURCE_DATA,
+                'payload': dict(EXAMPLE_SOURCE_DATA),
             },
             context={'request': self.request},
         )
@@ -218,7 +219,7 @@ def test__serializers__create_and_update(self):
         self.assertEqual(models.Entity.objects.count(), 0)
         submission.save()
         self.assertNotEqual(models.Entity.objects.count(), 0)
-        self.assertIn('aether_extractor_enrichment', submission.data['payload'])
+        self.assertIn(ENTITY_EXTRACTION_ENRICHMENT, submission.data['payload'])
 
         # check entity
         entity = serializers.EntitySerializer(
@@ -227,7 +228,7 @@ def test__serializers__create_and_update(self):
                 'submission': submission.data['id'],
                 'schemadecorator': schemadecorator.data['id'],
                 'status': 'Pending Approval',
-                'payload': EXAMPLE_SOURCE_DATA,  # has no id
+                'payload': dict(EXAMPLE_SOURCE_DATA),  # has no id
             },
             context={'request': self.request},
         )
@@ -306,7 +307,7 @@ def test__serializers__create_and_update(self):
 
         create_count = 6
         # make objects
-        payloads = [EXAMPLE_SOURCE_DATA_ENTITY for i in range(create_count)]
+        payloads = [dict(EXAMPLE_SOURCE_DATA_ENTITY) for i in range(create_count)]
         for pl in payloads:
             pl.update({'id': str(uuid.uuid4())})
         data = [
diff --git a/aether-kernel/aether/kernel/api/tests/test_utils.py b/aether-kernel/aether/kernel/api/tests/test_utils.py
index 00d2edb42..2319caf58 100644
--- a/aether-kernel/aether/kernel/api/tests/test_utils.py
+++ b/aether-kernel/aether/kernel/api/tests/test_utils.py
@@ -41,7 +41,7 @@ def setUp(self):
         username = 'test'
         email = 'test@example.com'
         password = 'testtest'
-        self.user = get_user_model().objects.create_user(username, email, password)
+        get_user_model().objects.create_user(username, email, password)
         self.assertTrue(self.client.login(username=username, password=password))
 
         self.project = models.Project.objects.create(
@@ -49,16 +49,16 @@ def setUp(self):
             name='a project name',
         )
 
-        url = reverse('project-artefacts', kwargs={'pk': self.project.pk})
+        self.mappingset_id = str(uuid.uuid4())
+        self.mapping_1 = str(uuid.uuid4())
+        self.mapping_2 = str(uuid.uuid4())
 
-        mappingset_id = uuid.uuid4()
-
-        data = {
+        artefacts = {
             'mappingsets': [{
+                'id': self.mappingset_id,
                 'name': 'Test Mappingset',
                 'input': EXAMPLE_SOURCE_DATA_WITH_LOCATION,
                 'schema': {},
-                'id': mappingset_id,
             }],
             'schemas': [
                 {
@@ -72,89 +72,63 @@ def setUp(self):
                 {
                     'name': 'Person',
                     'definition': EXAMPLE_SCHEMA,
-                }
+                },
             ],
             'mappings': [
                 {
+                    'id': self.mapping_1,
                     'name': 'mapping-1',
                     'definition': {
                         'mapping': EXAMPLE_FIELD_MAPPINGS,
                     },
                     'is_active': True,
                     'is_ready_only': False,
-                    'mappingset': mappingset_id,
+                    'mappingset': self.mappingset_id,
                 },
                 {
+                    'id': self.mapping_2,
                     'name': 'mapping-2',
                     'definition': {
                         'mapping': EXAMPLE_FIELD_MAPPINGS_LOCATION,
                     },
                     'is_active': True,
                     'is_ready_only': False,
-                    'mappingset': mappingset_id,
-                }
+                    'mappingset': self.mappingset_id,
+                },
             ],
         }
-        self.project_artefacts = self.client.patch(
-            url,
-            data=data,
+        self.client.patch(
+            reverse('project-artefacts', kwargs={'pk': self.project.pk}),
+            data=artefacts,
             content_type='application/json',
-        ).json()
+        )
 
     def tearDown(self):
         self.project.delete()
         self.client.logout()
 
     def test_get_unique_schemas_used(self):
-        url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]})
-        mapping = self.client.get(url).json()
-        if mapping['name'] == 'mapping-1':
-            mapping_1 = mapping
-            self.mapping = mapping['id']
-        else:
-            mapping_2 = mapping
-
-        url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][1]})
-        mapping = self.client.get(url).json()
-        if mapping['name'] == 'mapping-2':
-            mapping_2 = mapping
-        else:
-            mapping_1 = mapping
-            self.mapping = mapping['id']
-
-        self.assertEqual(mapping_2['name'], 'mapping-2')
-        self.assertEqual(mapping_1['name'], 'mapping-1')
-
-        result = utils.get_unique_schemas_used([mapping_1['id']])
+        result = utils.get_unique_schemas_used([self.mapping_1])
         self.assertEqual(len(result), 1)
         self.assertEqual(next(iter(result)), 'Person')
         self.assertFalse(result[next(iter(result))]['is_unique'])
 
-        result = utils.get_unique_schemas_used([mapping_2['id']])
+        result = utils.get_unique_schemas_used([self.mapping_2])
         self.assertEqual(len(result), 2)
         self.assertFalse(result['Person']['is_unique'])
         self.assertTrue(result['Location']['is_unique'])
 
-        result = utils.get_unique_schemas_used([mapping_2['id'], mapping_1['id']])
+        result = utils.get_unique_schemas_used([self.mapping_1, self.mapping_2])
         self.assertEqual(len(result), 2)
         self.assertTrue(result['Person']['is_unique'])
         self.assertTrue(result['Location']['is_unique'])
 
     def test_bulk_delete_by_mappings_mapping(self):
-        url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]})
-        mapping = self.client.get(url).json()
-        if mapping['name'] == 'mapping-2':
-            mapping_2 = mapping
-        else:
-            url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][1]})
-            mapping_2 = self.client.get(url).json()
-
-        self.assertEqual(mapping_2['name'], 'mapping-2')
         opts = {
             'entities': True,
             'schemas': True,
         }
-        result = utils.bulk_delete_by_mappings(opts, None, [mapping_2['id']])
+        result = utils.bulk_delete_by_mappings(opts, None, [self.mapping_2])
         self.assertFalse(result['schemas']['Person']['is_unique'])
         self.assertTrue(result['schemas']['Location']['is_unique'])
         self.assertTrue(result['schemas']['Location']['is_deleted'])
@@ -162,16 +136,12 @@ def test_bulk_delete_by_mappings_mapping(self):
         self.assertNotIn('submissions', result)
 
     def test_bulk_delete_by_mappings_mappingset(self):
-        url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]})
-        mapping = self.client.get(url).json()
-        mapping_object = models.Mapping.objects.get(pk=mapping['id'])
-        mappingset = mapping_object.mappingset.id
         opts = {
             'entities': True,
             'schemas': True,
             'submissions': True
         }
-        result = utils.bulk_delete_by_mappings(opts, mappingset)
+        result = utils.bulk_delete_by_mappings(opts, self.mappingset_id)
         self.assertTrue(result['schemas']['Person']['is_unique'])
         self.assertTrue(result['schemas']['Location']['is_unique'])
         self.assertTrue(result['schemas']['Location']['is_deleted'])
@@ -184,32 +154,30 @@ def test_bulk_delete_by_mappings_mappingset(self):
             'schemas': False,
             'submissions': False
         }
-        result = utils.bulk_delete_by_mappings(opts, mappingset)
+        result = utils.bulk_delete_by_mappings(opts, self.mappingset_id)
         self.assertEqual(result, {})
 
     def test_bulk_delete_by_mappings_with_submissions(self):
-        mapping_object = models.Mapping.objects.get(pk=self.project_artefacts['mappings'][0])
-        mappingset = mapping_object.mappingset.id
-
-        url = reverse('submission-list')
-        data = {
-            'payload': EXAMPLE_SOURCE_DATA_WITH_LOCATION,
+        submission = {
+            'payload': dict(EXAMPLE_SOURCE_DATA_WITH_LOCATION),
             'project': str(self.project.id),
-            'mappingset': mappingset
+            'mappingset': self.mappingset_id,
         }
         self.client.post(
-            url,
-            data=data,
+            reverse('submission-list'),
+            data=submission,
             content_type='application/json',
         )
+        entity_count = models.Entity.objects.filter(
+            mapping__id__in=[self.mapping_1, self.mapping_2]
+        ).count()
+        self.assertTrue(entity_count > 0)
+
         opts = {
             'entities': True,
             'submissions': True
         }
-        entity_count = models.Entity.objects.filter(
-            mapping__id__in=self.project_artefacts['mappings']
-        ).count()
-        result = utils.bulk_delete_by_mappings(opts, mappingset)
+        result = utils.bulk_delete_by_mappings(opts, self.mappingset_id)
         self.assertEqual(result['entities']['total'], entity_count)
         self.assertTrue(result['entities']['schemas'])
         self.assertEqual(result['entities']['schemas'][0]['name'], 'Person')
diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py
index 180dc1499..7c95ac848 100644
--- a/aether-kernel/aether/kernel/api/tests/test_views.py
+++ b/aether-kernel/aether/kernel/api/tests/test_views.py
@@ -26,10 +26,10 @@
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase, override_settings
 from django.urls import reverse
-from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
-
 from rest_framework import status
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
+
 from aether.kernel.api import models
 from aether.kernel.api.entity_extractor import run_entity_extraction
 from aether.kernel.api.tests.utils.generators import generate_project
@@ -99,7 +99,7 @@ def setUp(self):
         )
 
         self.submission = models.Submission.objects.create(
-            payload=EXAMPLE_SOURCE_DATA,
+            payload=dict(EXAMPLE_SOURCE_DATA),
             mappingset=self.mappingset,
             project=self.project,
         )
@@ -159,7 +159,7 @@ def test_project_stats_view(self):
                 # this will also trigger the entities extraction
                 # (4 entities per submission -> 3 for self.schemadecorator + 1 for schemadecorator_2)
                 self.helper_create_object('submission-list', {
-                    'payload': EXAMPLE_SOURCE_DATA,
+                    'payload': dict(EXAMPLE_SOURCE_DATA),
                     'mappingset': str(self.mappingset.pk),
                 })
 
@@ -772,7 +772,7 @@ def test_submission__extract__endpoint(self):
         models.Entity.objects.all().delete()  # remove all entities
         self.assertEqual(self.submission.entities.count(), 0)
         self.submission.refresh_from_db()
-        self.assertEqual(self.submission.payload['aether_errors'], [])
+        self.assertEqual(self.submission.payload[ENTITY_EXTRACTION_ERRORS], [])
 
         response = self.client.post(url)
         self.assertEqual(response.status_code, 405, 'only PATCH')
@@ -783,11 +783,29 @@ def test_submission__extract__endpoint(self):
         self.assertEqual(response.status_code, 400)
         self.assertEqual(self.submission.entities.count(), 0)
         self.submission.refresh_from_db()
-        self.assertEqual(self.submission.payload['aether_errors'], ['oops'])
+        self.assertEqual(self.submission.payload[ENTITY_EXTRACTION_ERRORS], ['oops'])
 
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(self.submission.entities.count(), 0)
+        entities_count = self.submission.entities.count()
+        self.assertNotEqual(entities_count, 0)
+        entity_ids = [e.id for e in self.submission.entities.all()]
+
+        # re-extract (same number of entities with the same IDs)
+        models.Entity.objects.all().delete()  # remove all entities
+        self.client.patch(url)
+        self.assertEqual(entities_count, self.submission.entities.count())
+        self.assertEqual(entity_ids, [e.id for e in self.submission.entities.all()])
+
+        # re-extract (no new Entities just updated)
+        for e in self.submission.entities.all():
+            e.status = 'Pending Approval'
+            e.save()
+        self.client.patch(url)
+        self.assertEqual(entities_count, self.submission.entities.count())
+        self.assertEqual(entity_ids, [e.id for e in self.submission.entities.all()])
+        for e in self.submission.entities.all():
+            self.assertEqual(e.status, 'Publishable')
 
     def test_schema_unique_usage(self):
         url = reverse('schema-unique-usage')
@@ -919,7 +937,7 @@ def test_submission_validate(self):
         url = reverse('submission-validate')
         data = {
             'mappingset': str(test_mappingset.id),
-            'payload': PAYLOAD
+            'payload': dict(PAYLOAD),
         }
         response = self.client.post(
             url,
@@ -947,7 +965,7 @@ def test_submission_validate(self):
         self.assertEqual('Not accessible by this realm', response_data['detail'])
 
         del PAYLOAD['facility_name']
-        data['payload'] = PAYLOAD
+        data['payload'] = dict(PAYLOAD)
         response = self.client.post(
             url,
             data=data,
@@ -971,7 +989,7 @@ def test_submission_validate(self):
 
         data = {
             'mappingset': 'wrong-uuid',
-            'payload': PAYLOAD
+            'payload': dict(PAYLOAD)
         }
         response = self.client.post(
             url,
diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py
index c3185c60b..862e32d4c 100644
--- a/aether-kernel/aether/kernel/api/views.py
+++ b/aether-kernel/aether/kernel/api/views.py
@@ -387,10 +387,10 @@ def validate(self, request, *args, **kwargs):
         expected response:
 
         {
-            #   Bool indicating if the submission is valid or not
+            # flag indicating if the submission is valid or not
             'is_valid': True|False,
 
-            #   list of entities successfully generated from the submitted payload
+            # list of entities successfully generated from the submitted payload
             'entities': [],
 
             # list of encountered errors
@@ -409,13 +409,11 @@ def validate(self, request, *args, **kwargs):
         try:
             mappingset = get_object_or_404(models.MappingSet.objects.all(), pk=mappingset_id)
         except Exception as e:
-            return Response(
-                str(e),
-                status=status.HTTP_400_BAD_REQUEST,
-            )
+            return Response(str(e), status=status.HTTP_400_BAD_REQUEST)
 
         if not self.check_realm_permission(request, mappingset):
             raise PermissionDenied(_('Not accessible by this realm'))
+
         mappings = mappingset.mappings.all()
         result = {
             'is_valid': True,
@@ -432,20 +430,20 @@ def validate(self, request, *args, **kwargs):
                     submission_payload=payload,
                     mapping_definition=mapping.definition,
                     schemas=schemas,
+                    mapping_id=mapping.id,
                 )
                 if submission_data.get(ENTITY_EXTRACTION_ERRORS):
                     result['is_valid'] = False
                     result[ENTITY_EXTRACTION_ERRORS] += submission_data[ENTITY_EXTRACTION_ERRORS]
                 else:
                     result['entities'] += entities
+
             except Exception as e:  # pragma: no cover
                 result['is_valid'] = False
                 result[ENTITY_EXTRACTION_ERRORS].append(str(e))
-                return Response(result, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-        if result['is_valid']:
-            return Response(result)
-        return Response(result, status=status.HTTP_400_BAD_REQUEST)
+        _status = status.HTTP_200_OK if result['is_valid'] else status.HTTP_400_BAD_REQUEST
+        return Response(result, status=_status)
 
     @action(detail=True, methods=['patch'])
     def extract(self, request, pk, *args, **kwargs):
@@ -823,9 +821,10 @@ def validate_mappings_view(request, *args, **kwargs):
 
     def run_mapping_validation(submission_payload, mapping_definition, schemas):
         submission_data, entities = extract_create_entities(
-            submission_payload,
-            mapping_definition,
-            schemas,
+            submission_payload=submission_payload,
+            mapping_definition=mapping_definition,
+            schemas=schemas,
+            mapping_id='validation',
         )
         validation_result = validate_mappings(
             submission_payload=submission_payload,
diff --git a/aether-kernel/aether/kernel/management/commands/extract_entities.py b/aether-kernel/aether/kernel/management/commands/extract_entities.py
index 8535b509f..79ebc11aa 100644
--- a/aether-kernel/aether/kernel/management/commands/extract_entities.py
+++ b/aether-kernel/aether/kernel/management/commands/extract_entities.py
@@ -21,9 +21,9 @@
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
 from aether.kernel.api.models import Submission, Entity
 from aether.kernel.api.entity_extractor import run_entity_extraction
-from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
 
 
 class Command(BaseCommand):
diff --git a/aether-kernel/aether/kernel/tests/test_commands.py b/aether-kernel/aether/kernel/tests/test_commands.py
index 892526012..f1899c003 100644
--- a/aether-kernel/aether/kernel/tests/test_commands.py
+++ b/aether-kernel/aether/kernel/tests/test_commands.py
@@ -23,7 +23,7 @@
 from django.test import TestCase
 
 from aether.kernel.api.tests.utils.generators import generate_project
-from aether.kernel.api.models import Submission, Entity
+from aether.kernel.api.models import Entity
 
 
 class ExtractEntitiesCommandTest(TestCase):
@@ -56,16 +56,16 @@ def test__extract_entities__success(self):
     def test__extract_entities__error(self):
         generate_project()
         self.assertNotEqual(Entity.objects.count(), 0)
-        entities = Entity.objects.count()
+        entities_count = Entity.objects.count()
 
-        for submission in Submission.objects.all():
-            submission.payload = {}  # make extraction fail
-            submission.save()
+        with mock.patch('aether.kernel.api.entity_extractor.extract_create_entities',
+                        side_effect=Exception('oops')) as mock_extractor:
+            try:
+                call_command('extract_entities', stdout=self.out, stderr=self.out)
+                self.assertTrue(True)
+            except Exception:
+                self.assertTrue(False)
 
-        try:
-            call_command('extract_entities', stdout=self.out, stderr=self.out)
-            self.assertTrue(True)
-        except Exception:
-            self.assertTrue(False)
-        self.assertEqual(Entity.objects.count(), entities,
+        self.assertEqual(Entity.objects.count(), entities_count,
                          'transaction atomic reverts the deletion')
+        mock_extractor.assert_called()
diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
index 3e2cee6df..d020969f4 100644
--- a/aether-kernel/conf/pip/requirements.txt
+++ b/aether-kernel/conf/pip/requirements.txt
@@ -12,7 +12,7 @@
 #
 ################################################################################
 
-aether.python==1.1.2
+aether.python==1.2.0
 aether.sdk==1.3.1
 attrs==19.3.0
 autopep8==1.5.4