National Computing Infrastructure (NCI), Site specific configuration (#…

…2281) Co-authored-by: Valeriu Predoi <valeriu.predoi@gmail.com>
ESMValGroup · Jan 19, 2024 · e40f672 · e40f672
1 parent e03d320
commit e40f672
Show file tree

Hide file tree

Showing 3 changed files with 133 additions and 0 deletions.
diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml
@@ -36,6 +36,7 @@ CMIP6:
     ESGF: '{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}'
     ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/'
     SYNDA: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}'
+    NCI: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}'
   input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc'
   output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}'
   cmor_type: 'CMIP6'
@@ -45,6 +46,7 @@ CMIP5:
   input_dir:
     default: '/'
     BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}'
+    NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}'
     BSC: '{type}/{project}/{exp}/{dataset.lower}'
     CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/'
     DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}'
@@ -64,6 +66,7 @@ CMIP3:
     DKRZ: '{exp}/{modeling_realm}/{frequency}/{short_name}/{dataset}/{ensemble}'
     ESGF: '{project.lower}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}'
     IPSL: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}/{short_name}'
+    NCI: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{latestversion}'
   input_file: '{short_name}_*.nc'
   output_file: '{project}_{institute}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}'
   cmor_type: 'CMIP3'

diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml
@@ -247,3 +247,23 @@ drs:
 #  OBS: default
 #  obs4MIPs: default
 #  ana4mips: BADC
+
+# Site-specific entries: NCI
+# Uncomment the lines below to locate data at NCI.
+#rootpath:
+#  CMIP6: [/g/data/oi10/replicas/CMIP6, /g/data/fs38/publications/CMIP6, /g/data/xp65/public/apps/esmvaltool/replicas/CMIP6]
+#  CMIP5: [/g/data/r87/DRSv3/CMIP5, /g/data/al33/replicas/CMIP5/combined, /g/data/rr3/publications/CMIP5/output1, /g/data/xp65/public/apps/esmvaltool/replicas/cmip5/output1]
+#  CMIP3: /g/data/r87/DRSv3/CMIP3
+#  OBS: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2
+#  OBS6: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2
+#  obs4MIPs: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2
+#  ana4mips: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2
+#  native6: /g/data/xp65/public/apps/esmvaltool/native6
+#
+#drs:
+#  CMIP6: NCI
+#  CMIP5: NCI
+#  CMIP3: NCI
+#  CORDEX: ESGF
+#  obs4MIPs: default
+#  ana4mips: default
diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml
@@ -452,6 +452,84 @@ get_input_filelist:
       - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc
     found_files:
       - historical/Amon/ta/HadGEM2-ES/r1i1p1/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+
+  - drs: NCI
+    variable:
+      <<: *variable
+      timerange: '1980/2002'
+    available_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+    dirs:
+      - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta
+    file_patterns:
+      - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc
+    found_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+
+  - drs: NCI
+    variable:
+      <<: *variable
+      timerange: '2000/2005'
+      version: v20110329
+    available_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+    dirs:
+      - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta
+    file_patterns:
+      - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc
+    found_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+
+  - drs: NCI
+    variable:
+      <<: *variable
+      ensemble: '*'
+      timerange: '2000/2005'
+    available_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc
+    dirs:
+      - INPE/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta
+    file_patterns:
+      - ta_Amon_HadGEM2-ES_historical_**.nc
+    found_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc
+
+  - drs: NCI
+    variable:
+      <<: *variable
+      timerange: '1980/2002'
+    available_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmo1/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+    available_symlinks:
+      - link_name: MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest
+        target: v20120928
+    dirs:
+      - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta
+    file_patterns:
+      - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc
+    found_files:
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+      - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
 
   # Test other projects
 
@@ -515,6 +593,38 @@ get_input_filelist:
     found_files:
       - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_195001-199912.nc
       - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_200001-200112.nc
+
+  - drs: NCI
+    variable:
+      variable_group: test
+      short_name: ta
+      original_short_name: ta
+      dataset: HadGEM3-GC31-LL
+      activity: CMIP
+      project: CMIP6
+      institute: [MOHC, NERC]
+      frequency: mon
+      modeling_realm: [atmos]
+      mip: Amon
+      exp: historical
+      grid: gn
+      ensemble: r1i1p1f1
+      timerange: '1999/2000'
+      diagnostic: test_diag
+      preprocessor: test_preproc
+    available_files:
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_190001-194912.nc
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_190001-194912.nc
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_195001-199912.nc
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_200001-201412.nc
+    dirs:
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/
+      - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/
+    file_patterns:
+      - ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc
+    found_files:
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_195001-199912.nc
+      - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_200001-201412.nc
 
   - drs: default
     variable: