MIN: restructure odim.py/gamic.py, add test_odim.py/test_gamic.py (#154)

* MIN: restructure odim.py, add test_odim.py

* FIX: use dim0 - property

* FIX: test DeprecationWarning, remove superfluous code

* FIX: fix and remove superfluous code

* FIX: Check UserWarning

* MNT: add _H5NetCDFMetadata Class with common methods/properties for ODIM/GAMIC, add gamic tests

* MNT: fix datetime tests

* DOC: add history.md entry

* MNT: remove superfluous code and add more tests, add fsspec to dev requirements

ci/unittests.yml

@@ -7,6 +7,7 @@ dependencies:
   - cmweather
   - coverage
   - dask
+  - fsspec
   - h5netcdf
   - h5py
   - lat_lon_parser

docs/history.md

@@ -3,6 +3,7 @@
 ## Development Version (unreleased)
 
 * MNT: Update GitHub actions, address DeprecationWarnings ({pull}`153`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
+* MNT: restructure odim.py/gamic.py, add test_odim.py/test_gamic.py ({pull}`154`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 
 ## 0.4.3 (2024-02-24)
 

requirements.txt

@@ -1,7 +1,7 @@
 cmweather
 dask
-h5netcdf
-h5py
+h5netcdf >= 1.0.0
+h5py >= 3.0.0
 lat_lon_parser
 netCDF4
 numpy

requirements_dev.txt

@@ -9,3 +9,4 @@ twine
 pytest
 black
 isort
+fsspec

tests/io/test_gamic.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# Copyright (c) 2024, openradar developers.
+# Distributed under the MIT License. See LICENSE for more info.
+
+"""Tests for `io.backends.gamic` module.
+
+ported from wradlib
+"""
+
+import numpy as np
+import pytest
+
+from xradar.io.backends import gamic
+
+
+def create_ray_header(nrays=360):
+    dtype = {
+        "names": [
+            "azimuth_start",
+            "azimuth_stop",
+            "elevation_start",
+            "elevation_stop",
+            "timestamp",
+        ],
+        "formats": ["<f8", "<f8", "<f8", "<f8", "<i8"],
+        "offsets": [0, 8, 16, 24, 32],
+        "itemsize": 40,
+    }
+    ray_header = np.zeros(nrays, dtype=dtype)
+    start_az = np.linspace(0, 360, nrays, endpoint=False, dtype=np.float64)
+    stop_az = np.linspace(1, 361, nrays, endpoint=False, dtype=np.float64)
+    ray_header["azimuth_start"] = start_az
+    ray_header["azimuth_stop"] = stop_az
+
+    start_el = np.ones_like(start_az, dtype=np.float64)
+    stop_el = np.ones_like(start_az, dtype=np.float64)
+    ray_header["elevation_start"] = start_el
+    ray_header["elevation_stop"] = stop_el
+
+    time = np.arange(
+        1527831788042000, 1527831788042000 + nrays * 100000, 100000, dtype=np.int64
+    )
+    ray_header["timestamp"] = time
+    return ray_header
+
+
+@pytest.mark.parametrize("nrays", [180, 240, 360, 720])
+def test_get_azimuth(nrays):
+    ray_header = create_ray_header(nrays)
+    actual = gamic._get_azimuth(ray_header)
+    udiff = np.unique(np.diff(actual))
+    assert len(actual) == nrays
+    assert len(udiff) == 1
+    assert udiff[0] == 360.0 / nrays
+
+
+@pytest.mark.parametrize("nrays", [180, 240, 360, 720])
+def test_get_elevation(nrays):
+    ray_header = create_ray_header(nrays)
+    actual = gamic._get_elevation(ray_header)
+    unique = np.unique(actual)
+    assert len(actual) == nrays
+    assert len(unique) == 1
+    assert unique[0] == 1.0
+
+
+@pytest.mark.parametrize("nrays", [180, 240, 360, 720])
+def test_get_timestamp(nrays):
+    ray_header = create_ray_header(nrays)
+    actual = gamic._get_time(ray_header)
+    udiff = np.unique(np.diff(actual))
+    assert len(actual) == nrays
+    assert len(udiff) == 1
+    assert udiff[0] == 100000
+
+
+@pytest.mark.parametrize(
+    "ang",
+    [("elevation", "azimuth"), ("azimuth", "elevation")],
+)
+def test_get_fixed_dim_and_angle(ang):
+    how = {ang[0]: 1.0}
+    dim, angle = gamic._get_fixed_dim_and_angle(how)
+    assert dim == ang[1]
+    assert angle == 1.0
+
+
+@pytest.mark.parametrize("range_step", [100, 150, 300, 1000])
+@pytest.mark.parametrize("range_samples", [1, 2, 4])
+def test_get_range(range_step, range_samples):
+    where = dict(bin_count=10, range_step=range_step, range_samples=range_samples)
+    rng, cent_first, bin_range = gamic._get_range(where)
+    assert len(rng) == 10
+    assert np.unique(np.diff(rng))[0] == range_step * range_samples
+    assert cent_first == (range_step * range_samples) / 2
+    assert bin_range == range_step * range_samples
+
+
+def test_GamicH5NetCDFMetadata(gamic_file):
+    store = gamic.GamicStore.open(gamic_file, group="sweep_0")
+    with pytest.warns(DeprecationWarning):
+        assert store.root.first_dim == "azimuth"

tests/io/test_io.py

@@ -4,9 +4,11 @@
 
 """Tests for `io` module."""
 
+import io
 import tempfile
 
 import datatree
+import fsspec
 import h5py
 import numpy as np
 import pytest
@@ -98,6 +100,16 @@ def test_open_cfradial1_dataset(cfradial1_file):
     assert ds.sweep_number == 8
 
 
+@pytest.mark.parametrize("sweep", ["sweep_0", 0, [0, 1], ["sweep_0", "sweep_1"]])
+def test_open_odim_datatree_sweep(odim_file, sweep):
+    dtree = open_odim_datatree(odim_file, sweep=sweep)
+    if isinstance(sweep, (str, int)):
+        lswp = len([sweep])
+    else:
+        lswp = len(sweep)
+    assert len(dtree.groups[1:]) == lswp
+
+
 def test_open_odim_datatree(odim_file):
     dtree = open_odim_datatree(odim_file)
 
@@ -173,34 +185,79 @@ def test_open_odim_datatree(odim_file):
         assert ds.sweep_number.values == int(grp[7:])
 
 
-def test_open_odim_dataset(odim_file):
+@pytest.mark.parametrize("first_dim", ["auto", "time"])
+@pytest.mark.parametrize("fix_second_angle", [False, True])
+def test_open_odim_dataset(odim_file, first_dim, fix_second_angle):
     # open first sweep group
-    ds = xr.open_dataset(odim_file, group="sweep_0", engine="odim")
-    assert dict(ds.sizes) == {"azimuth": 360, "range": 1200}
+    ds = xr.open_dataset(
+        odim_file,
+        group="sweep_0",
+        engine="odim",
+        first_dim=first_dim,
+        fix_second_angle=fix_second_angle,
+    )
+    dim0 = "time" if first_dim == "time" else "azimuth"
+    assert dict(ds.sizes) == {dim0: 360, "range": 1200}
     assert set(ds.data_vars) & (
         sweep_dataset_vars | non_standard_sweep_dataset_vars
     ) == {"WRADH", "VRADH", "PHIDP", "DBZH", "RHOHV", "KDP", "TH", "ZDR"}
     assert ds.sweep_number == 0
 
     # open last sweep group
-    ds = xr.open_dataset(odim_file, group="sweep_11", engine="odim")
-    assert dict(ds.sizes) == {"azimuth": 360, "range": 280}
-    assert set(ds.data_vars) & (
-        sweep_dataset_vars | non_standard_sweep_dataset_vars
-    ) == {"VRADH", "KDP", "WRADH", "TH", "RHOHV", "PHIDP", "ZDR", "DBZH"}
-    assert ds.sweep_number == 11
-
-    # open last sweep group, auto
     ds = xr.open_dataset(
         odim_file,
         group="sweep_11",
         engine="odim",
-        backend_kwargs=dict(first_dim="time"),
+        first_dim=first_dim,
+        fix_second_angle=fix_second_angle,
     )
-    assert dict(ds.sizes) == {"time": 360, "range": 280}
+    assert dict(ds.sizes) == {dim0: 360, "range": 280}
+    assert set(ds.data_vars) & (
+        sweep_dataset_vars | non_standard_sweep_dataset_vars
+    ) == {"VRADH", "KDP", "WRADH", "TH", "RHOHV", "PHIDP", "ZDR", "DBZH"}
     assert ds.sweep_number == 11
 
 
+def test_open_odim_dataset_stream(odim_file):
+    with open(odim_file, mode="rb") as fhandle:
+        contents = io.BytesIO(fhandle.read())
+        xr.open_dataset(contents, group="sweep_0", engine="odim")
+
+
+def test_open_odim_dataset_fsspec(odim_file):
+    with fsspec.open(odim_file, mode="rb") as fhandle:
+        xr.open_dataset(fhandle, group="sweep_0", engine="odim")
+
+
+def test_open_odim_store(odim_file):
+    store = xradar.io.backends.odim.OdimStore.open(odim_file, group="sweep_0")
+    assert store.substore[0].root.a1gate == 86
+    assert store.substore[0].root.site_coords == (
+        151.20899963378906,
+        -33.700801849365234,
+        195.0,
+    )
+    assert store.substore[0].root._get_site_coords() == (
+        151.20899963378906,
+        -33.700801849365234,
+        195.0,
+    )
+    assert store.substore[0].root.sweep_fixed_angle == 0.5
+    assert store.substore[0].root._get_time() == np.datetime64(
+        "2018-12-20T06:06:28", "s"
+    )
+
+
+@pytest.mark.parametrize("sweep", ["sweep_0", 0, [0, 1], ["sweep_0", "sweep_1"]])
+def test_open_gamic_datatree_sweep(gamic_file, sweep):
+    dtree = open_gamic_datatree(gamic_file, sweep=sweep)
+    if isinstance(sweep, (str, int)):
+        lswp = len([sweep])
+    else:
+        lswp = len(sweep)
+    assert len(dtree.groups[1:]) == lswp
+
+
 def test_open_gamic_datatree(gamic_file):
     dtree = open_gamic_datatree(gamic_file)
 
@@ -281,10 +338,19 @@ def test_open_gamic_datatree(gamic_file):
         assert ds.sweep_number == i
 
 
-def test_open_gamic_dataset(gamic_file):
+@pytest.mark.parametrize("first_dim", ["auto", "time"])
+@pytest.mark.parametrize("fix_second_angle", [False, True])
+def test_open_gamic_dataset(gamic_file, first_dim, fix_second_angle):
     # open first sweep group
-    ds = xr.open_dataset(gamic_file, group="sweep_0", engine="gamic")
-    assert dict(ds.sizes) == {"azimuth": 361, "range": 360}
+    ds = xr.open_dataset(
+        gamic_file,
+        group="sweep_0",
+        engine="gamic",
+        first_dim=first_dim,
+        fix_second_angle=fix_second_angle,
+    )
+    dim0 = "time" if first_dim == "time" else "azimuth"
+    assert dict(ds.sizes) == {dim0: 361, "range": 360}
     assert set(ds.data_vars) & (
         sweep_dataset_vars | non_standard_sweep_dataset_vars
     ) == {
@@ -304,8 +370,14 @@ def test_open_gamic_dataset(gamic_file):
     assert ds.sweep_number == 0
 
     # open last sweep group
-    ds = xr.open_dataset(gamic_file, group="sweep_9", engine="gamic")
-    assert dict(ds.sizes) == {"azimuth": 360, "range": 1000}
+    ds = xr.open_dataset(
+        gamic_file,
+        group="sweep_9",
+        engine="gamic",
+        first_dim=first_dim,
+        fix_second_angle=fix_second_angle,
+    )
+    assert dict(ds.sizes) == {dim0: 360, "range": 1000}
     assert set(ds.data_vars) & (
         sweep_dataset_vars | non_standard_sweep_dataset_vars
     ) == {
@@ -324,15 +396,24 @@ def test_open_gamic_dataset(gamic_file):
     }
     assert ds.sweep_number == 9
 
-    # open last sweep group, auto
-    ds = xr.open_dataset(
-        gamic_file,
-        group="sweep_9",
-        engine="gamic",
-        backend_kwargs=dict(first_dim="time"),
-    )
-    assert dict(ds.sizes) == {"time": 360, "range": 1000}
-    assert ds.sweep_number == 9
+
+def test_open_gamic_dataset_stream(gamic_file):
+    with open(gamic_file, mode="rb") as fhandle:
+        contents = io.BytesIO(fhandle.read())
+        xr.open_dataset(contents, group="sweep_9", engine="gamic")
+
+
+def test_open_gamic_dataset_fsspec(gamic_file):
+    with fsspec.open(gamic_file, mode="rb") as fhandle:
+        xr.open_dataset(fhandle, group="sweep_9", engine="gamic")
+
+
+def test_open_gamic_store(gamic_file):
+    store = xradar.io.backends.gamic.GamicStore.open(gamic_file, group="sweep_0")
+    assert store.root.site_coords == (6.4569489, 50.9287272, 310.0)
+    assert store.root._get_site_coords() == (6.4569489, 50.9287272, 310.0)
+    assert store.root.sweep_fixed_angle == 28.0
+    assert store.root._get_time() == np.datetime64("2018-06-01T05:40:47.041000", "us")
 
 
 def test_open_gamic_dataset_reindex(gamic_file):