290 align test datasets & Broken cordex datastore (#300)

* WIP * WIP * Ready to try live translator tests on Gadi * docstring * Fixed e2e workflow, updated cordex translator & test data * Formatting
ACCESS-NRI · Dec 10, 2024 · b0fcefa · b0fcefa
1 parent 6391c5c
commit b0fcefa
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 23 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -18,6 +18,6 @@ jobs:
           key: ${{secrets.DEPLOY_KEY}}
           script: |
             cd ${{secrets.GADI_REPO_PATH}}
-            qsub bin/build_all.sh
+            qsub bin/test_end_to_end.sh
 
             
diff --git a/src/access_nri_intake/catalog/translators.py b/src/access_nri_intake/catalog/translators.py
@@ -385,7 +385,9 @@ def __init__(self, source, columns):
 
         super().__init__(source, columns)
         self.set_dispatch(
-            input_name="source_id", core_colname="model", func=super()._model_translator
+            input_name="project_id",
+            core_colname="model",
+            func=super()._model_translator,
         )
         self.set_dispatch(
             input_name="variable_id",

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -25,6 +25,11 @@ def config_dir():
     return Path(here / "e2e/configs")
 
 
+@fixture(scope="session")
+def live_config_dir():
+    return Path(here).parent / "config"
+
+
 @fixture(scope="session")
 def BASE_DIR(tmp_path_factory):
     yield tmp_path_factory.mktemp("catalog-dir")

diff --git a/tests/data/esm_datastore/cordex-ig45.csv b/tests/data/esm_datastore/cordex-ig45.csv
@@ -1,6 +1,6 @@
-path,file_type,project_id,resolution,institution_id,source_id,experiment_id,member_id,frequency,variable_id,version,time_range
-/g/data/ig45/QldFCP-2/output/CMIP6/DD/AUS-10i/UQ-DEC/ACCESS-CM2/ssp126/r2i1p1f1/CCAMoc-v2112/v1-r1/day/hus200/v20240709/hus200_AUS-10i_ACCESS-CM2_ssp126_r2i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_day_20580101-20581231.nc,f,output,AUS-10i,UQ-DEC,ACCESS-CM2,ssp126,r2i1p1f1,day,hus200,v20240709,20580101-20581231
-/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/ACCESS-ESM1-5/ssp126/r20i1p1f1/CCAMoc-v2112/v1-r1/mon/va925/v20240722/va925_AUS-20i_ACCESS-ESM1-5_ssp126_r20i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_mon_208101-209012.nc,f,CORDEX,AUS-20i,UQ-DEC,ACCESS-ESM1-5,ssp126,r20i1p1f1,mon,va925,v20240722,208101-209012
-/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/ACCESS-ESM1-5/ssp370/r6i1p1f1/CCAM-v2105/v1-r1/mon/clh/v20240722/clh_AUS-20i_ACCESS-ESM1-5_ssp370_r6i1p1f1_UQ-DEC_CCAM-v2105_v1-r1_mon_201501-202012.nc,f,CORDEX,AUS-20i,UQ-DEC,ACCESS-ESM1-5,ssp370,r6i1p1f1,mon,clh,v20240722,201501-202012
-/g/data/ig45/QldFCP-2/output/CMIP6/DD/AUS-10i/UQ-DEC/ACCESS-CM2/ssp126/r2i1p1f1/CCAMoc-v2112/v1-r1/day/ta850/v20240709/ta850_AUS-10i_ACCESS-CM2_ssp126_r2i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_day_20340101-20341231.nc,f,output,AUS-10i,UQ-DEC,ACCESS-CM2,ssp126,r2i1p1f1,day,ta850,v20240709,20340101-20341231
-/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/NorESM2-MM/ssp126/r1i1p1f1/CCAMoc-v2112/v1-r1/mon/hus200/v20240722/hus200_AUS-20i_NorESM2-MM_ssp126_r1i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_mon_201501-202012.nc,f,CORDEX,AUS-20i,UQ-DEC,NorESM2-MM,ssp126,r1i1p1f1,mon,hus200,v20240722,201501-202012
+path,file_type,project_id,experiment_id,member_id,frequency,variable_id,version,time_range
+/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/ACCESS-ESM1-5/ssp126/r20i1p1f1/CCAMoc-v2112/v1-r1/mon/va925/v20240722/va925_AUS-20i_ACCESS-ESM1-5_ssp126_r20i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_mon_208101-209012.nc,f,ACCESS-ESM1-5,ssp126,r20i1p1f1,mon,va925,v20240722,208101-209012
+/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/ACCESS-ESM1-5/ssp370/r6i1p1f1/CCAM-v2105/v1-r1/mon/clh/v20240722/clh_AUS-20i_ACCESS-ESM1-5_ssp370_r6i1p1f1_UQ-DEC_CCAM-v2105_v1-r1_mon_201501-202012.nc,f,ACCESS-ESM1-5,ssp370,r6i1p1f1,mon,clh,v20240722,201501-202012
+/g/data/ig45/QldFCP-2/CORDEX/CMIP6/DD/AUS-20i/UQ-DEC/NorESM2-MM/ssp126/r1i1p1f1/CCAMoc-v2112/v1-r1/mon/hus200/v20240722/hus200_AUS-20i_NorESM2-MM_ssp126_r1i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_mon_201501-202012.nc,f,NorESM2-MM,ssp126,r1i1p1f1,mon,hus200,v20240722,201501-202012
+/g/data/ig45/QldFCP-2/output/CMIP6/DD/AUS-10i/UQ-DEC/ACCESS-CM2/ssp126/r2i1p1f1/CCAMoc-v2112/v1-r1/day/hus200/v20240709/hus200_AUS-10i_ACCESS-CM2_ssp126_r2i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_day_20580101-20581231.nc,f,ACCESS-CM2,ssp126,r2i1p1f1,day,hus200,v20240709,20580101-20581231
+/g/data/ig45/QldFCP-2/output/CMIP6/DD/AUS-10i/UQ-DEC/ACCESS-CM2/ssp126/r2i1p1f1/CCAMoc-v2112/v1-r1/day/ta850/v20240709/ta850_AUS-10i_ACCESS-CM2_ssp126_r2i1p1f1_UQ-DEC_CCAMoc-v2112_v1-r1_day_20340101-20341231.nc,f,ACCESS-CM2,ssp126,r2i1p1f1,day,ta850,v20240709,20340101-20341231
diff --git a/tests/data/esm_datastore/cordex-ig45.json b/tests/data/esm_datastore/cordex-ig45.json
@@ -11,9 +11,6 @@
         "groupby_attrs": [
             "file_type",
             "project_id",
-            "resolution",
-            "institution_id",
-            "source_id",
             "experiment_id",
             "member_id",
             "frequency",
@@ -39,15 +36,6 @@
         {
             "column_name": "project_id"
         },
-        {
-            "column_name": "resolution"
-        },
-        {
-            "column_name": "institution_id"
-        },
-        {
-            "column_name": "source_id"
-        },
         {
             "column_name": "experiment_id"
         },

diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py
@@ -0,0 +1,5 @@
+import pytest
+
+e2e = pytest.mark.skipif(
+    "not config.getoption('--e2e')",
+)
diff --git a/tests/e2e/test_datasets_representative.py b/tests/e2e/test_datasets_representative.py
@@ -0,0 +1,50 @@
+import pytest
+import yaml
+
+import access_nri_intake.catalog.translators as translators
+from access_nri_intake.cli import build
+
+from . import e2e
+
+
+@e2e
+@pytest.mark.parametrize(
+    "translator_name",
+    [
+        t
+        for t in dir(translators)
+        if t.endswith("Translator") and not t.startswith("Default")
+    ],
+)
+def test_alignment(translator_name, live_config_dir, BASE_DIR, v_num):
+    # Now live test the translator. Honestly, might be overkill - it might be easier
+    # to just extract the json files, open them, check they match the test data
+    filenames = [f for f in live_config_dir.glob("*.yaml")]
+    # Now we want to open them & throw away anything where builder != null.
+    translator_fnames = []
+
+    for fname in filenames:
+        with open(fname) as fo:
+            catalog_metadata = yaml.load(fo, yaml.FullLoader)
+            if catalog_metadata["translator"] == translator_name:
+                translator_fnames.append(str(fname))
+
+    assert len(translator_fnames) == 1
+
+    try:
+        build(
+            [
+                *translator_fnames,
+                "--build_base_path",
+                str(BASE_DIR),
+                "--catalog_base_path",
+                "./",
+                "--catalog_file",
+                "metacatalog.csv",
+                "--version",
+                v_num,
+                "--no_update",
+            ]
+        )
+    except Exception as exc:
+        assert False, f"Failed to build {translator_name} with exception {exc}"
diff --git a/tests/e2e/test_end_to_end.py b/tests/e2e/test_end_to_end.py
@@ -6,9 +6,7 @@
 
 from access_nri_intake.cli import build
 
-e2e = pytest.mark.skipif(
-    "not config.getoption('--e2e')",
-)
+from . import e2e
 
 
 @e2e