[IMPORT] Embed reusable import module into GeoNature core (#2833)

* feat(import): integrate import module (backend and frontend) in GeoNature core
* feat(import): add requirement in occhab import
* feat(import): add inter-field condition between different entities
* feat(import, error) : create an "enum" to list import error
* feat(check) parent and unique column to entity
* feat: import schema
* feat: import to synthese
* feat: import to occhab
* feat(import): remove ng_module from import
* fix(destination): add type occhab to module occhab
* feat(imports): several checks made generic
* feat(imports): add parent/child related checks
* feat(import/occhab): add id_station_source
* feat(imports/occhab): add id_import FK in pr_occhab
* feat(back): list all imports
* feat(import): access to import report from several places
* feat(import): download file and delete import from imports list
* feat(import): add statistics display in import list
* feat(import list, statistic): enable generic tooltip in the import list
* feat(import, mapping): skip content mapping if not necessary
* feat(mapping, import): add field to field condition
* feat(import, plot): add generic plot in backend import
* feat(import, doc): add documentation from gn_module_import
* feat(import): add frontend test
* feat(import, synthese): add the possibility to declare jdd in the import file 
* feat(model, import): Prohibit deletion of import with stations including habitats not originating from the same import.
* feat(import, occhab): add default mapping (SINP + GEONATURE)
* feat (mapping, import) : hide jdd selection form
* feat(import, report): update import report 
* feat(report, synthese): add id_import to synthese
* feat(report) link from import report to occhab, and filter on id_import

---------

Co-authored-by: jacquesfize <jacques.fize@ecrins-parcnational.fr>
Co-authored-by: Pierre Narcisi <pierre.narcisi@mnhn.fr>
Co-authored-by: Andria Capai <andria_capai@natural-solutions.eu>
Co-authored-by: VincentCauchois <vincent.cauchois@mnhn.fr>
Co-authored-by: Etienne Delclaux <etienne_delclaux@natural-solutions.eu>
Co-authored-by: Élie Bouttier <bouttier@users.noreply.github.com>
Co-authored-by: Julien Corny <julien_corny@natural-solutions.eu>

.github/workflows/cypress.yml

@@ -5,12 +5,13 @@ on:
       - master
       - hotfixes
       - develop
-      - fixtestfront
+      - feat/import
   pull_request:
     branches:
       - master
       - hotfixes
       - develop
+      - feat/import
 
 jobs:
   mount_app_and_run_cypress:
@@ -31,6 +32,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
       - name: Add postgis_raster database extension
@@ -110,12 +120,18 @@ jobs:
           geonature db upgrade occhab-samples@head
           geonature install-gn-module contrib/gn_module_validation VALIDATION --build=false
           geonature permissions supergrant --group --nom "Grp_admin" --yes
+          geonature db upgrade import-samples@head
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Run GeoNature backend
         run: geonature dev_back &
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
+      - name: Run celery
+        run: celery -A geonature.celery_app:app worker &
+        working-directory: ./backend/geonature/
+        env:
+          GEONATURE_CONFIG_FILE: "${{ github.workspace }}/config/test_config.toml"
       - name: Cypress run
         uses: cypress-io/github-action@v5
         with:

.github/workflows/pytest.yml

@@ -15,7 +15,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
     strategy:
       fail-fast: false
       matrix:
@@ -46,6 +45,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
       - uses: actions/checkout@v4

backend/geonature/app.py

@@ -141,7 +141,6 @@ def create_app(with_external_mods=True):
         from geonature.utils.celery import celery_app
 
         celery_app.init_app(app)
-        celery_app.conf.update(app.config["CELERY"])
 
     # Emails configuration
     if app.config["MAIL_CONFIG"]:
@@ -218,6 +217,7 @@ def set_sentry_context():
         ("geonature.core.gn_profiles.routes:routes", "/gn_profiles"),
         ("geonature.core.sensitivity.routes:routes", None),
         ("geonature.core.notifications.routes:routes", "/notifications"),
+        ("geonature.core.imports.blueprint:blueprint", "/import"),
     ]:
         module_name, blueprint_name = blueprint_path.split(":")
         blueprint = getattr(import_module(module_name), blueprint_name)

backend/geonature/celery_app.py

@@ -1,6 +1,7 @@
 from .app import create_app
 from .utils.celery import celery_app as app
 from .utils.module import iter_modules_dist
+from .utils.env import db
 
 
 flask_app = create_app()
@@ -9,7 +10,9 @@
 class ContextTask(app.Task):
     def __call__(self, *args, **kwargs):
         with flask_app.app_context():
-            return self.run(*args, **kwargs)
+            result = self.run(*args, **kwargs)
+            db.session.remove()
+        return result
 
 
 app.Task = ContextTask

backend/geonature/core/gn_commons/routes.py

@@ -68,6 +68,7 @@ def list_modules():
     query = (
         select(TModules)
         .options(joinedload(TModules.objects))
+        .options(joinedload(TModules.destination))
         .where(TModules.module_code.notin_(exclude))
         .order_by(TModules.module_order.asc())
         .order_by(TModules.module_label.asc())
@@ -80,7 +81,7 @@ def list_modules():
         # HACK : on a besoin d'avoir le module GeoNature en front pour l'URL de la doc
         if module.module_code == "GEONATURE":
             module_allowed = True
-        module_dict = module.as_dict(fields=["objects"])
+        module_dict = module.as_dict(fields=["objects", "destination.code"])
         # TODO : use has_any_permissions instead - must refactor the front
         module_dict["cruved"] = {
             action: get_scope(action, module_code=module.module_code, bypass_warning=True)

backend/geonature/core/gn_meta/routes.py

@@ -26,7 +26,6 @@
 from geonature.utils.env import DB, db
 from geonature.core.gn_synthese.models import (
     Synthese,
-    TSources,
     CorAreaSynthese,
 )
 from geonature.core.gn_permissions.decorators import login_required
@@ -244,14 +243,9 @@ def uuid_report():
         select(Synthese)
         .where(Synthese.id_module == id_module if id_module is not None else True)
         .where(Synthese.id_dataset == ds_id if ds_id is not None else True)
+        .where(Synthese.id_import == id_import if id_import is not None else True)
     )
 
-    # TODO test in module import ?
-    if id_import:
-        query = query.outerjoin(TSources, TSources.id_source == Synthese.id_source).where(
-            TSources.name_source == f"Import(id={id_import})"
-        )
-
     query = query.order_by(Synthese.id_synthese)
 
     data = [
@@ -323,13 +317,9 @@ def sensi_report(ds_id=None):
         .where(LAreas.id_type == func.ref_geo.get_id_area_type("DEP"))
         .where(Synthese.id_module == id_module if id_module else True)
         .where(Synthese.id_dataset == ds_id)
+        .where(Synthese.id_import == id_import if id_import else True)
     )
 
-    if id_import:
-        query = query.outerjoin(TSources, TSources.id_source == Synthese.id_source).where(
-            TSources.name_source == "Import(id={})".format(id_import)
-        )
-
     query = query.group_by(
         Synthese.id_synthese, TNomenclatures.cd_nomenclature, TNomenclatures.label_fr
     )

backend/geonature/core/gn_monitoring/routes.py

@@ -79,6 +79,7 @@ def get_site_areas(id_site):
     params = request.args
 
     query = (
+        # TODO@LAreas.geom_4326
         select(corSiteArea, func.ST_Transform(LAreas.geom, 4326))
         .join(LAreas, LAreas.id_area == corSiteArea.c.id_area)
         .where(corSiteArea.c.id_base_site == id_site)

backend/geonature/core/gn_permissions/tools.py

@@ -4,7 +4,7 @@
 
 import sqlalchemy as sa
 from sqlalchemy.orm import joinedload
-from flask import g
+from flask import has_request_context, g
 
 from geonature.core.gn_commons.models import TModules
 from geonature.core.gn_permissions.models import (
@@ -50,9 +50,12 @@ def _get_user_permissions(id_role):
 def get_user_permissions(id_role=None):
     if id_role is None:
         id_role = g.current_user.id_role
-    if id_role not in g._permissions_by_user:
-        g._permissions_by_user[id_role] = _get_user_permissions(id_role)
-    return g._permissions_by_user[id_role]
+    if has_request_context():
+        if id_role not in g._permissions_by_user:
+            g._permissions_by_user[id_role] = _get_user_permissions(id_role)
+        return g._permissions_by_user[id_role]
+    else:
+        return _get_user_permissions(id_role)
 
 
 def _get_permissions(id_role, module_code, object_code, action_code):
@@ -93,9 +96,12 @@ def get_permissions(action_code, id_role=None, module_code=None, object_code=Non
             object_code = "ALL"
 
     ident = (id_role, module_code, object_code, action_code)
-    if ident not in g._permissions:
-        g._permissions[ident] = _get_permissions(*ident)
-    return g._permissions[ident]
+    if has_request_context():
+        if ident not in g._permissions:
+            g._permissions[ident] = _get_permissions(*ident)
+        return g._permissions[ident]
+    else:
+        return _get_permissions(*ident)
 
 
 def get_scope(action_code, id_role=None, module_code=None, object_code=None, bypass_warning=False):

backend/geonature/core/gn_synthese/imports/__init__.py

@@ -0,0 +1 @@
+