Merge pull request #83 from legend-exp/dev

Bug fix: partially filled `VectorOfVectors` buffers in LH5 read operations are invalid

src/lgdo/lh5/_serializers/read/vector_of_vectors.py

@@ -168,6 +168,12 @@ def _h5_read_vector_of_vectors(
     )
 
     if obj_buf is not None:
+        # if the buffer is partially filled, cumulative_length will be invalid
+        # (i.e. non monotonically increasing). Let's fix that but filling the
+        # rest of the array with the length of flattened_data
+        end = obj_buf_start + n_rows_read
+        obj_buf.cumulative_length.nda[end:] = obj_buf.cumulative_length.nda[end - 1]
+
         return obj_buf, n_rows_read
 
     return (

src/lgdo/types/table.py

@@ -327,13 +327,19 @@ def eval(
                 else:
                     self_unwrap[obj] = flat_self[obj].view_as("np", with_units=False)
 
+        msg = f"evaluating {expr!r} with locals={(self_unwrap | parameters)} and {has_ak=}"
+        log.debug(msg)
+
         # use numexpr if we are only dealing with numpy data types
         if not has_ak:
             out_data = ne.evaluate(
                 expr,
                 local_dict=(self_unwrap | parameters),
             )
 
+            msg = f"...the result is {out_data!r}"
+            log.debug(msg)
+
             # need to convert back to LGDO
             # np.evaluate should always return a numpy thing?
             if out_data.ndim == 0:
@@ -353,6 +359,9 @@ def eval(
         globs = {"ak": ak, "np": np}
         out_data = eval(expr, globs, (self_unwrap | parameters))
 
+        msg = f"...the result is {out_data!r}"
+        log.debug(msg)
+
         # need to convert back to LGDO
         if isinstance(out_data, ak.Array):
             if out_data.ndim == 1:

tests/conftest.py

@@ -26,5 +26,5 @@ def pytest_sessionfinish(session, exitstatus):  # noqa: ARG001
 @pytest.fixture(scope="session")
 def lgnd_test_data():
     ldata = LegendTestData()
-    ldata.checkout("5097863")
+    ldata.checkout("8f55832")
     return ldata

tests/types/test_table_eval.py

@@ -22,6 +22,7 @@ def test_eval_dependency():
                 ],
             ),
             "e": lgdo.VectorOfVectors([[1, 2, 3], [4], [], [8, 6]]),
+            "ee": lgdo.VectorOfVectors([[[1], [2, 3]], [[], [4]], [[]], [[8, 6]]]),
             "tbl": lgdo.Table(
                 col_dict={
                     "z": lgdo.Array([1, 1, 1, 1]),
@@ -60,6 +61,10 @@ def test_eval_dependency():
     assert isinstance(r, lgdo.VectorOfVectors)
     assert r == lgdo.VectorOfVectors([[3, 5, 7], [9], [], [17, 13]])
 
+    r = obj.eval("2*ee + 1")
+    assert isinstance(r, lgdo.VectorOfVectors)
+    assert r == lgdo.VectorOfVectors([[[3], [5, 7]], [[], [9]], [[]], [[17, 13]]])
+
     r = obj.eval("e > 2")
     assert isinstance(r, lgdo.VectorOfVectors)
     assert r == lgdo.VectorOfVectors([[False, False, True], [True], [], [True, True]])

tests/types/test_vectorofvectors.py

@@ -11,7 +11,7 @@
 import pytest
 
 import lgdo
-from lgdo import Array, VectorOfVectors
+from lgdo import Array, VectorOfVectors, lh5
 
 VovColl = namedtuple("VovColl", ["v2d", "v3d", "v4d"])
 
@@ -414,3 +414,13 @@ def test_view_as(testvov):
     assert ak.is_valid(ak_arr)
     assert len(ak_arr) == len(v3d)
     assert ak.all(ak_arr == [[[1, 2], [3, 4, 5]], [[2], [4, 8, 9, 7]], [[5, 3, 1]]])
+
+
+def test_lh5_iterator_view_as(lgnd_test_data):
+    it = lh5.LH5Iterator(
+        lgnd_test_data.get_path("lh5/l200-p03-r000-phy-20230312T055349Z-tier_psp.lh5"),
+        "ch1067205/dsp/energies",
+    )
+
+    for obj, _, _ in it:
+        assert ak.is_valid(obj.view_as("ak"))