Merge pull request #22 from openimis/v1.3

V1.3

.github/workflows/openmis-module-test.yml

@@ -0,0 +1,100 @@
+name: Automated CI testing
+# This workflow run automatically for every commit on github it checks the syntax and launch the tests.
+# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      comment:
+        description: Just a simple comment to know the purpose of the manual build
+        required: false
+
+jobs:
+  run_test:
+    runs-on: ubuntu-latest
+    services:
+      mssql:
+        image: mcr.microsoft.com/mssql/server:2017-latest
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: GitHub999
+        ports:
+          - 1433:1433
+        # needed because the mssql container does not provide a health check
+        options: --health-interval=10s --health-timeout=3s --health-start-period=10s --health-retries=10 --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1"
+
+    steps:
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: install linux packages
+        run: |
+          wget https://raw.githubusercontent.com/openimis/database_ms_sqlserver/main/Empty%20databases/openIMIS_ONLINE.sql -O openIMIS_ONLINE.sql
+          wget https://raw.githubusercontent.com/openimis/database_ms_sqlserver/main/Demo%20database/openIMIS_demo_ONLINE.sql -O openIMIS_demo_ONLINE.sql
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+          curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
+          sudo apt-get update
+          sudo ACCEPT_EULA=Y apt-get install -y  mssql-tools build-essential dialog apt-utils unixodbc-dev -y
+          python -m pip install --upgrade pip
+      - name: pull openimis backend
+        run: |
+          rm ./openimis -rf
+          git clone --depth 1 --branch develop https://github.com/openimis/openimis-be_py.git ./openimis
+      - name: copy current branch
+        uses: actions/checkout@v2
+        with:
+          path: './current-module'
+      - name: Update the configuration
+        working-directory: ./openimis
+        run: |
+          export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
+          echo "the local module called $MODULE_NAME will be injected in openIMIS .json"
+          jq --arg name "$MODULE_NAME" 'if [.modules[].name  ==  ($name)]| max then (.modules[]  | select(.name == ($name)) | .pip)|="../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json
+          echo $(jq --arg name "$MODULE_NAME" 'if [.modules[].name  ==  ($name)]| max then (.modules[]  | select(.name == ($name)) | .pip)|="../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json)  > openimis.json
+      - name: Install openIMIS Python dependencies
+        working-directory: ./openimis
+        run: |
+          pip install -r requirements.txt
+          python modules-requirements.py openimis.json > modules-requirements.txt
+          cat modules-requirements.txt
+          pip install -r modules-requirements.txt
+      - name: Environment info
+        working-directory: ./openimis
+        run: |
+          pip list
+      - name: Initialize DB
+        run: |
+          /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -Q 'DROP DATABASE IF EXISTS imis'
+          /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -Q 'CREATE DATABASE imis'
+          /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -d imis -i openIMIS_ONLINE.sql | grep . | uniq -c
+          /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -d imis -i openIMIS_demo_ONLINE.sql | grep . | uniq -c
+        env:
+          SA_PASSWORD: GitHub999
+          ACCEPT_EULA: Y
+
+#      - name: Check formatting with black
+#        run: |
+#          black --check .
+
+      - name: Django tests
+        working-directory: ./openimis/openIMIS
+        run: |
+          export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
+          python -V
+          ls -l
+          python manage.py migrate
+          python init_test_db.py | grep . | uniq -c
+          python manage.py test --keepdb $MODULE_NAME
+        env:
+          SECRET_KEY: secret
+          DEBUG: true
+          #DJANGO_SETTINGS_MODULE: hat.settings
+          DB_HOST: localhost
+          DB_PORT: 1433
+          DB_NAME: imis
+          DB_USER: sa
+          DB_PASSWORD: GitHub999
+          #DEV_SERVER: true
+          SITE_ROOT: api
+          REMOTE_USER_AUTHENTICATION: True

.github/workflows/python-publish.yml

@@ -13,6 +13,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - uses: olegtarasov/get-tag@v2.1
+      id: tagName
     - uses: actions/checkout@v2
     - name: Set up Python
       uses: actions/setup-python@v2
@@ -21,11 +23,16 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install setuptools wheel twine jq
+        
+    - name: update setup.py
+      run: |
+        echo "tag to use $GIT_TAG_NAME"
+        sed -i "s/version='.*'/version='$GIT_TAG_NAME'/g" setup.py
     - name: Build and publish
       env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{secrets.PYPI_TOKEN}}
       run: |
         python setup.py sdist bdist_wheel
-        twine upload dist/*
+        twine upload dist/*

insuree/apps.py

@@ -1,4 +1,6 @@
 from django.apps import AppConfig
+from django.conf import settings
+
 
 MODULE_NAME = "insuree"
 
@@ -19,6 +21,9 @@
     "excluded_insuree_chfids": ['999999999'],  # fake insurees (and bound families) used, for example, in 'funding'
     "renewal_photo_age_adult": 60,  # age (in months) of a picture due for renewal for adults
     "renewal_photo_age_child": 12,  # age (in months) of a picture due for renewal for children
+    "insuree_number_validator": None,  # Insuree number *function* that validates the insuree number
+    "insuree_number_length": None,  # Insuree number length to validate
+    "insuree_number_modulo_root": None,  # modulo base for checksum on last digit, requires length to be set too
 }
 
 
@@ -42,6 +47,9 @@ class InsureeConfig(AppConfig):
     excluded_insuree_chfids = ['999999999']
     renewal_photo_age_adult = 60
     renewal_photo_age_child = 12
+    insuree_number_validator = None
+    insuree_number_length = None
+    insuree_number_modulo_root = None
 
     def _configure_permissions(self, cfg):
         InsureeConfig.gql_query_insurees_perms = cfg["gql_query_insurees_perms"]
@@ -56,6 +64,9 @@ def _configure_permissions(self, cfg):
         InsureeConfig.gql_mutation_update_insurees_perms = cfg["gql_mutation_update_insurees_perms"]
         InsureeConfig.gql_mutation_delete_insurees_perms = cfg["gql_mutation_delete_insurees_perms"]
         InsureeConfig.insuree_photos_root_path = cfg["insuree_photos_root_path"]
+        InsureeConfig.insuree_number_validator = cfg["insuree_number_validator"]
+        InsureeConfig.insuree_number_length = cfg["insuree_number_length"]
+        InsureeConfig.insuree_number_modulo_root = cfg["insuree_number_modulo_root"]
 
     def _configure_fake_insurees(self, cfg):
         InsureeConfig.excluded_insuree_chfids = cfg["excluded_insuree_chfids"]
@@ -64,10 +75,25 @@ def _configure_renewal(self, cfg):
         InsureeConfig.renewal_photo_age_adult = cfg["renewal_photo_age_adult"]
         InsureeConfig.renewal_photo_age_child = cfg["renewal_photo_age_child"]
 
-
     def ready(self):
         from core.models import ModuleConfiguration
         cfg = ModuleConfiguration.get_or_default(MODULE_NAME, DEFAULT_CFG)
         self._configure_permissions(cfg)
         self._configure_fake_insurees(cfg)
         self._configure_renewal(cfg)
+
+    # Getting these at runtime for easier testing
+    @classmethod
+    def get_insuree_number_validator(cls):
+        return cls.insuree_number_validator or \
+               (hasattr(settings, "INSUREE_NUMBER_VALIDATOR") and settings.INSUREE_NUMBER_VALIDATOR)
+
+    @classmethod
+    def get_insuree_number_length(cls):
+        return cls.insuree_number_length or \
+               (hasattr(settings, "INSUREE_NUMBER_LENGTH") and settings.INSUREE_NUMBER_LENGTH)
+
+    @classmethod
+    def get_insuree_number_modulo_root(cls):
+        return cls.insuree_number_modulo_root or\
+               (hasattr(settings, "INSUREE_NUMBER_MODULE_ROOT") and settings.INSUREE_NUMBER_MODULE_ROOT)

insuree/gql_mutations.py

@@ -4,6 +4,8 @@
 import base64
 from copy import copy
 import graphene
+from insuree.services import validate_insuree_number
+
 from .apps import InsureeConfig
 from core import filter_validity, assert_string_length
 from core.schema import TinyInt, SmallInt, OpenIMISMutation
@@ -83,6 +85,8 @@ class FamilyBase:
     confirmation_type_id = graphene.String(max_length=3, required=False)
     json_ext = graphene.types.json.JSONString(required=False)
 
+    contribution = graphene.types.json.JSONString(required=False)
+
     head_insuree = graphene.Field(FamilyHeadInsureeInputType, required=False)
 
 
@@ -160,7 +164,7 @@ def handle_insuree_photo(user, now, insuree, data):
     photo_bin = data.get('photo', None)
     if photo_bin and InsureeConfig.insuree_photos_root_path and photo_bin != insuree_photo.photo:
         (file_dir, file_name) = create_file(now, insuree.id, photo_bin)
-        data.pop('photo')
+        data.pop('photo', None)
         data['folder'] = file_dir
         data['filename'] = file_name
 
@@ -179,16 +183,14 @@ def handle_insuree_photo(user, now, insuree, data):
 
 
 def update_or_create_insuree(data, user):
-    if "client_mutation_id" in data:
-        data.pop('client_mutation_id')
-    if "client_mutation_label" in data:
-        data.pop('client_mutation_label')
+    data.pop('client_mutation_id', None)
+    data.pop('client_mutation_label', None)
     photo = data.pop('photo', None)
     from core import datetime
     now = datetime.datetime.now()
     data['audit_user_id'] = user.id_for_audit
     data['validity_from'] = now
-    insuree_uuid = data.pop('uuid') if 'uuid' in data else None
+    insuree_uuid = data.pop('uuid', None)
     # update_or_create(uuid=insuree_uuid, ...)
     # doesn't work because of explicit attempt to set null to uuid!
     if insuree_uuid:
@@ -199,7 +201,11 @@ def update_or_create_insuree(data, user):
         reset_insuree_before_update(insuree)
         [setattr(insuree, key, data[key]) for key in data]
     else:
-        insuree = Insuree.objects.create(**data)
+        errors = validate_insuree_number(data["chf_id"])
+        if errors:
+            raise Exception("Invalid insuree number")
+        else:
+            insuree = Insuree.objects.create(**data)
     insuree.save()
     photo = handle_insuree_photo(user, now, insuree, photo)
     if photo:
@@ -222,15 +228,13 @@ def reset_family_before_update(family):
 
 
 def update_or_create_family(data, user):
-    if "client_mutation_id" in data:
-        data.pop('client_mutation_id')
-    if "client_mutation_label" in data:
-        data.pop('client_mutation_label')
+    data.pop('client_mutation_id', None)
+    data.pop('client_mutation_label', None)
     head_insuree_data = data.pop('head_insuree')
     head_insuree_data["head"] = True
     head_insuree = update_or_create_insuree(head_insuree_data, user)
     data["head_insuree"] = head_insuree
-    family_uuid = data.pop('uuid') if 'uuid' in data else None
+    family_uuid = data.pop('uuid', None)
     # update_or_create(uuid=family_uuid, ...)
     # doesn't work because of explicit attempt to set null to uuid!
     if family_uuid:
@@ -241,6 +245,7 @@ def update_or_create_family(data, user):
         reset_family_before_update(family)
         [setattr(family, key, data[key]) for key in data]
     else:
+        data.pop('contribution', None)
         family = Family.objects.create(**data)
     family.save()
     head_insuree.family = family
@@ -270,10 +275,15 @@ def async_mutate(cls, user, **data):
             from core.utils import TimeUtils
             data['validity_from'] = TimeUtils.now()
             client_mutation_id = data.get("client_mutation_id")
+            # Validate insuree number right away
+            errors = validate_insuree_number(data.get("head_insuree", {}).get("chf_id", None))
+            if errors:
+                return errors
             family = update_or_create_family(data, user)
             FamilyMutation.object_mutated(user, client_mutation_id=client_mutation_id, family=family)
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_create_family")
             return [{
                 'message': _("insuree.mutation.failed_to_create_family"),
                 'detail': str(exc)}
@@ -304,8 +314,9 @@ def async_mutate(cls, user, **data):
             FamilyMutation.object_mutated(user, client_mutation_id=client_mutation_id, family=family)
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_update_family")
             return [{
-                'message': _("insuree.mutation.failed_to_create_family"),
+                'message': _("insuree.mutation.failed_to_update_family"),
                 'detail': str(exc)}
             ]
 
@@ -326,6 +337,7 @@ def set_family_deleted(family, delete_members):
         family.delete_history()
         return []
     except Exception as exc:
+        logger.exception("insuree.mutation.failed_to_delete_family")
         return {
             'title': family.uuid,
             'list': [{
@@ -389,10 +401,15 @@ def async_mutate(cls, user, **data):
             from core.utils import TimeUtils
             data['validity_from'] = TimeUtils.now()
             client_mutation_id = data.get("client_mutation_id")
+            # Validate insuree number right away
+            errors = validate_insuree_number(data.get("chf_id", None))
+            if errors:
+                return errors
             insuree = update_or_create_insuree(data, user)
             InsureeMutation.object_mutated(user, client_mutation_id=client_mutation_id, insuree=insuree)
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_create_insuree")
             return [{
                 'message': _("insuree.mutation.failed_to_create_insuree"),
                 'detail': str(exc)}
@@ -423,6 +440,7 @@ def async_mutate(cls, user, **data):
             InsureeMutation.object_mutated(user, client_mutation_id=client_mutation_id, insuree=insuree)
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_update_insuree")
             return [{
                 'message': _("insuree.mutation.failed_to_update_insuree"),
                 'detail': str(exc)}
@@ -435,6 +453,7 @@ def set_insuree_deleted(insuree):
         [ip.delete_history() for ip in insuree.insuree_policies.filter(validity_to__isnull=True)]
         return []
     except Exception as exc:
+        logger.exception("insuree.mutation.failed_to_delete_insuree")
         return {
             'title': insuree.chf_id,
             'list': [{
@@ -494,6 +513,7 @@ def cancel_policies(insuree):
         InsureePolicy.objects.bulk_update(ips, ['expiry_date'])
         return []
     except Exception as exc:
+        logger.exception("insuree.mutation.failed_to_cancel_insuree_policies")
         return {
             'title': insuree.chf_id,
             'list': [{
@@ -509,6 +529,7 @@ def remove_insuree(insuree):
         insuree.save()
         return []
     except Exception as exc:
+        logger.exception("insuree.mutation.failed_to_remove_insuree")
         return {
             'title': insuree.chf_id,
             'list': [{
@@ -592,6 +613,7 @@ def async_mutate(cls, user, **data):
             insuree.save()
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_set_head_insuree")
             return [{
                 'message': _("insuree.mutation.failed_to_set_head_insuree"),
                 'detail': str(exc)}
@@ -625,6 +647,7 @@ def async_mutate(cls, user, **data):
                 return cancel_policies(insuree)
             return None
         except Exception as exc:
+            logger.exception("insuree.mutation.failed_to_change_insuree_family")
             return [{
                 'message': _("insuree.mutation.failed_to_change_insuree_family"),
                 'detail': str(exc)}