Update xclim (#414)

## Overview Changes: * `finch` is now compatible with the latest `xclim` (v0.53.2) and `xscen` (v0.10.1). * Python 3.12 is now supported; Python 3.10+ coding conventions are now standard. * Python 3.8 and 3.9 support has been dropped. * The tempfile management for testing has been significantly improved. ## Additional Information Once this PR is merged and the cookiecutter is updated, we should be able to tag a new version.
bird-house · Dec 18, 2024 · d64e0b9 · d64e0b9
2 parents 372a548 + 8745a29
commit d64e0b9
Show file tree

Hide file tree

Showing 30 changed files with 518 additions and 297 deletions.
diff --git a/.codacy.yml b/.codacy.yml
@@ -1,8 +1,8 @@
 ---
 engines:
- pylint:
-   enabled: true
-   python_version: 3
+  pylint:
+    enabled: true
+    python_version: 3
 exclude_paths:
   - 'tests/**'
   - 'docs/source/conf.py'
diff --git a/.github/workflows/docker-testing.yml b/.github/workflows/docker-testing.yml
@@ -27,6 +27,7 @@ jobs:
           allowed-endpoints: >
             auth.docker.io:443
             conda.anaconda.org:443
+            files.pythonhosted.org:443
             github.com:443
             production.cloudflare.docker.com:443
             pypi.org:443

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -44,8 +44,8 @@ jobs:
       run:
         shell: bash -l {0}
     strategy:
-        matrix:
-            python-version: [ "3.9", "3.10", "3.11" ] # "3.12"
+      matrix:
+        python-version: [ "3.10", "3.11", "3.12" ]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
     rev: v3.17.0
     hooks:
       - id: pyupgrade
-        args: [ '--py39-plus' ]
+        args: [ '--py310-plus' ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
@@ -30,7 +30,7 @@ repos:
     rev: 24.10.0
     hooks:
       - id: black
-        args: [ '--target-version=py39' ]
+        args: [ '--target-version=py310' ]
   - repo: https://github.com/pycqa/flake8
     rev: 7.1.1
     hooks:
@@ -46,10 +46,10 @@ repos:
     rev: 1.8.7
     hooks:
       - id: nbqa-black
-        args: [ '--target-version=py39' ]
+        args: [ '--target-version=py310' ]
         additional_dependencies: [ 'black==24.8.0' ]
       - id: nbqa-pyupgrade
-        args: [ '--py39-plus' ]
+        args: [ '--py310-plus' ]
         additional_dependencies: [ 'pyupgrade==3.17.0' ]
       - id: nbqa-isort
         args: [ '--settings-file=setup.cfg' ]

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -13,12 +13,12 @@ sphinx:
   fail_on_warning: false
 
 # Optionally build your docs in additional formats such as PDF and ePub
-formats: []
+formats: [ ]
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-lts-latest
   tools:
-    python: "mambaforge-22.9"
+    python: "mambaforge-latest"
 
 conda:
   environment: environment-docs.yml

diff --git a/.yamllint.yaml b/.yamllint.yaml
@@ -1,8 +1,39 @@
 ---
 
 rules:
+
+  brackets:
+    forbid: false
+    min-spaces-inside: 1
+    max-spaces-inside: 1
+
+  commas:
+    min-spaces-after: 1
+
   document-start: disable
+
+  float-values:
+    require-numeral-before-decimal: true
+
+  hyphens:
+    max-spaces-after: 1
+
+  indentation:
+    indent-sequences: whatever
+    spaces: consistent
+
+  key-duplicates:
+    forbid-duplicated-merge-keys: true
+
   line-length:
-    max: 120
+    allow-non-breakable-words: true
+    allow-non-breakable-inline-mappings: true
+    max: 140
     level: warning
+
+  new-lines:
+    type: unix
+
+  trailing-spaces: {}
+
   truthy: disable
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -10,9 +10,10 @@ v0.13.0 (unreleased)
 * Permissions have been set for all workflows.
 * Pinned `cf-xarray`` below v0.9.3.
 * Reformatted and renamed `CHANGES.rst` to `CHANGELOG.rst`.
-* Dropped support for Python 3.8. The supported versions are now Python 3.9, 3.10, and 3.11.
+* Dropped support for Python 3.8 and 3.9. The supported versions are now Python 3.10, 3.11 and 3.12.
 * `black` has been updated to v2024.8.0, and coding conventions have been updated to Python3.9+.
 * Several dependencies now are pinned to baseline versions, including `anyascii`, `dask`, `ipython`, `matplotlib`, `nbsphinx` `numpy`, and `sphinxcontrib-bibtex`.
+* Added `xscen` dependency. Now used in spatial averaging of ensemble processes.
 
 v0.12.1 (2024-06-25)
 --------------------

diff --git a/environment-docs.yml b/environment-docs.yml
@@ -2,15 +2,16 @@ name: finch
 channels:
   - conda-forge
 dependencies:
-  - python >=3.8,<3.12
-  - anyascii >=0.3.2
+  - python >=3.10,<3.13
+  - anyascii >=0.3.0
   - birdy >=0.8.1
   - ipython >=8.0.0
   - matplotlib-base >=3.5.0
   - nbsphinx >=0.9.5
-  - pandas >=1.5.3,<2.2.0
+  - pandas >=2.2.0
   - pywps >=4.5.1
+  - setuptools >=71.0.0
   - sphinx >=7.0.0
   - sphinxcontrib-bibtex >=2.6.0
-  - xarray >=2023.01.0,<2023.11.0
-  - xclim =0.43 # remember to match xclim version in requirements_docs.txt as well
+  - xarray >=2023.11.0
+  - xclim ==0.52.2 # remember to match xclim version in requirements_docs.txt as well
diff --git a/environment.yml b/environment.yml
@@ -2,29 +2,29 @@ name: finch
 channels:
   - conda-forge
 dependencies:
-  - python >=3.9,<3.12
+  - python >=3.10,<3.13
   - pip >=24.2.0
   - anyascii >=0.3.0
-  - cftime
-  - cf_xarray >=0.6.1,<0.9.3
+  - cftime >=1.4.1
+  - cf_xarray >=0.9.3
   - click >=8.0.0
   - clisops >=0.11.0
   - dask >=2023.5.1
   - distributed
   - geopandas !=0.13.1
-  - h5netcdf
   - netcdf4 <=1.6.5
   - numpy >=1.23.0
-  - pandas >=1.5.3,<2.2.0
+  - pandas >=2.2.0
   - parse
   - psutil
   - python-slugify
   - pywps >=4.5.1
-  - pyyaml
-  - requests>=2.32.2
+  - pyyaml >=6.0.1
   - scipy >=1.9.0
   - sentry-sdk
+  - setuptools >=71.0.0
   - siphon
-  - xarray >=2023.01.0,<2023.11.0
-  - xclim =0.43  # remember to match xclim version in requirements_docs.txt as well
-  - xesmf >=0.8.2
+  - xarray >=2023.11.0
+  - xclim =0.52.2  # remember to match xclim version in requirements_docs.txt as well
+  - xesmf >=0.8.2,!=0.8.8
+  - xscen
diff --git a/finch/datasets.yml b/finch/datasets.yml
@@ -6,8 +6,8 @@ candcs-u5:
   suffix: "*.nc"
   pattern: "{variable}_{frequency}_BCCAQv2+ANUSPLIN300_{model}_{scenario}_{realization}_{date_start}-{date_end}.nc"
   allowed_values:
-    scenario: [rcp26, rcp45, rcp85]
-    variable: [tasmin, tasmax, pr]
+    scenario: [ rcp26, rcp45, rcp85 ]
+    variable: [ tasmin, tasmax, pr ]
     model:
       &u5models
       - BNU-ESM
@@ -39,18 +39,18 @@ candcs-u5:
     24models: *u5models
     pcic12:
     # taken from: https://www.pacificclimate.org/data/statistically-downscaled-climate-scenarios
-      - [ ACCESS1-0 , r1i1p1 ]
-      - [ CCSM4 , r2i1p1 ]
-      - [ CNRM-CM5 , r1i1p1 ]
-      - [ CSIRO-Mk3-6-0 , r1i1p1 ]
-      - [ CanESM2 , r1i1p1 ]
-      - [ GFDL-ESM2G , r1i1p1 ]
-      - [ HadGEM2-CC , r1i1p1 ]
-      - [ HadGEM2-ES , r1i1p1 ]
-      - [ MIROC5 , r3i1p1 ]
-      - [ MPI-ESM-LR , r3i1p1 ]
-      - [ MRI-CGCM3 , r1i1p1 ]
-      - [ inmcm4 , r1i1p1 ]
+      - [ ACCESS1-0, r1i1p1 ]
+      - [ CCSM4, r2i1p1 ]
+      - [ CNRM-CM5, r1i1p1 ]
+      - [ CSIRO-Mk3-6-0, r1i1p1 ]
+      - [ CanESM2, r1i1p1 ]
+      - [ GFDL-ESM2G, r1i1p1 ]
+      - [ HadGEM2-CC, r1i1p1 ]
+      - [ HadGEM2-ES, r1i1p1 ]
+      - [ MIROC5, r3i1p1 ]
+      - [ MPI-ESM-LR, r3i1p1 ]
+      - [ MRI-CGCM3, r1i1p1 ]
+      - [ inmcm4, r1i1p1 ]
 
 # This is a copy of the above for backward compatibility reasons.
 bccaqv2: *candcsu5
@@ -62,7 +62,7 @@ candcs-u6:
   pattern: "{variable}_{frequency}_BCCAQv2+ANUSPLIN300_{model}_{scenario}_{realization}_{}_{date_start}-{date_end}.nc"
   allowed_values:
     scenario: [ ssp126, ssp245, ssp585 ]
-    variable: [ tasmin, tasmax, pr]
+    variable: [ tasmin, tasmax, pr ]
     model:
       - ACCESS-CM2
       - ACCESS-ESM1-5
@@ -92,61 +92,61 @@ candcs-u6:
       - UKESM1-0-LL
   model_lists:
     26models:
-        - ACCESS-CM2
-        - ACCESS-ESM1-5
-        - BCC-CSM2-MR
-        - CMCC-ESM2
-        - CNRM-CM6-1
-        - CNRM-ESM2-1
-        - CanESM5
-        - [EC-Earth3, r4i1p1f1]
-        - EC-Earth3-Veg
-        - FGOALS-g3
-        - GFDL-ESM4
-        - HadGEM3-GC31-LL
-        - INM-CM4-8
-        - INM-CM5-0
-        - IPSL-CM6A-LR
-        - [KACE-1-0-G, r2i1p1f1]
-        - KIOST-ESM
-        - MIROC-ES2L
-        - MIROC6
-        - MPI-ESM1-2-HR
-        - MPI-ESM1-2-LR
-        - MRI-ESM2-0
-        - NorESM2-LM
-        - NorESM2-MM
-        - TaiESM1
-        - UKESM1-0-LL
+      - ACCESS-CM2
+      - ACCESS-ESM1-5
+      - BCC-CSM2-MR
+      - CMCC-ESM2
+      - CNRM-CM6-1
+      - CNRM-ESM2-1
+      - CanESM5
+      - [ EC-Earth3, r4i1p1f1 ]
+      - EC-Earth3-Veg
+      - FGOALS-g3
+      - GFDL-ESM4
+      - HadGEM3-GC31-LL
+      - INM-CM4-8
+      - INM-CM5-0
+      - IPSL-CM6A-LR
+      - [ KACE-1-0-G, r2i1p1f1 ]
+      - KIOST-ESM
+      - MIROC-ES2L
+      - MIROC6
+      - MPI-ESM1-2-HR
+      - MPI-ESM1-2-LR
+      - MRI-ESM2-0
+      - NorESM2-LM
+      - NorESM2-MM
+      - TaiESM1
+      - UKESM1-0-LL
 
 humidex-daily:
   local: false
   depth: 1
   path: https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/eccc/CCCS_humidex/Humidex/daily/catalog.html
   pattern: "{variable}_MBCn_ERA5-Land_{model}_{scenario}_{realization}_{date_start}-{date_end}.nc"
   allowed_values:
-    scenario: [ssp126, ssp245, ssp585]
-    variable: [HXmax]
+    scenario: [ ssp126, ssp245, ssp585 ]
+    variable: [ HXmax ]
     model:
       &hxmax_day_models
-        - MPI-ESM1-2-HR
-        - EC-Earth3
-        - ACCESS-CM2
-        - MIROC-ES2L
-        - EC-Earth3-Veg
-        - IPSL-CM6A-LR
-        - INM-CM5-0
-        - FGOALS-g3
-        - CanESM5
-        - MIROC6
-        - GISS-E2-1-G
-        - EC-Earth3-Veg-LR
-        - CNRM-ESM2-1
-        - MPI-ESM1-2-LR
-        - INM-CM4-8
-        - MRI-ESM2-0
-        - CMCC-ESM2
-        - ACCESS-ESM1-5
-        - CNRM-CM6-1
+      - MPI-ESM1-2-HR
+      - EC-Earth3
+      - ACCESS-CM2
+      - MIROC-ES2L
+      - EC-Earth3-Veg
+      - IPSL-CM6A-LR
+      - INM-CM5-0
+      - FGOALS-g3
+      - CanESM5
+      - MIROC6
+      - GISS-E2-1-G
+      - EC-Earth3-Veg-LR
+      - CNRM-ESM2-1
+      - MPI-ESM1-2-LR
+      - INM-CM4-8
+      - MRI-ESM2-0
+      - CMCC-ESM2
+      - ACCESS-ESM1-5
+      - CNRM-CM6-1
   model_lists:
     humidex_models: *hxmax_day_models
diff --git a/finch/processes/__init__.py b/finch/processes/__init__.py
@@ -1,6 +1,5 @@
 # noqa: D104
 import logging
-from pathlib import Path
 
 from pywps.configuration import get_config_value
 from xclim.core.indicator import registry as xclim_registry
@@ -52,11 +51,20 @@ def filter_func(elem):
 
 not_implemented = [
     "DC",
+    "E_SAT",
     "FWI",
+    "HURS",
+    "HURS_FROMDEWPOINT",
+    "HUSS",
+    "HUSS_FROMDEWPOINT",
     "RH",
     "RH_FROMDEWPOINT",
-    "E_SAT",
-    "HUSS",
+    "SPI",
+    "SPEI",
+    "WIND_POWER_POTENTIAL",
+    "WIND_PROFILE",
+    "WIND_SPEED_FROM_VECTOR",
+    "WIND_VECTOR_FROM_SPEED",
 ]
 
 
@@ -135,7 +143,7 @@ def get_processes():
         SubsetPolygonProcess(),
         AveragePolygonProcess(),
         HourlyToDailyProcess(),
-        GeoseriesToNetcdfProcess(),
+        # GeoseriesToNetcdfProcess(),
     ]
 
     return processes