From c4afb649cb67c186f20ac15d306193acbbc7a010 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Tue, 11 Jul 2023 11:48:04 -0700 Subject: [PATCH 1/2] Fix pre-commit errors. --- dask_deltatable/types.py | 6 +++--- dask_deltatable/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dask_deltatable/types.py b/dask_deltatable/types.py index ae74c6e..12eccdf 100644 --- a/dask_deltatable/types.py +++ b/dask_deltatable/types.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, List, Tuple, Union +from typing import Any, Union -Filter = Tuple[str, str, Any] -Filters = Union[List[Filter], List[List[Filter]], None] +Filter = tuple[str, str, Any] +Filters = Union[list[Filter], list[list[Filter]], None] diff --git a/dask_deltatable/utils.py b/dask_deltatable/utils.py index 58bd0d1..3901f63 100644 --- a/dask_deltatable/utils.py +++ b/dask_deltatable/utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, cast +from typing import cast from .types import Filter, Filters @@ -34,7 +34,7 @@ def get_partition_filters( return None if isinstance(filters[0][0], str): - filters = cast(List[List[Filter]], [filters]) + filters = cast(list[list[Filter]], [filters]) allowed_ops = { "=": "=", From e55af4fd45ef46091f97db5543b766c1b8d4bce0 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Tue, 11 Jul 2023 17:57:20 -0700 Subject: [PATCH 2/2] Test with partition_freq. --- tests/test_write.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index 314092b..e434772 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -24,7 +24,8 @@ False, ], ) -def test_roundtrip(tmpdir, with_index): +@pytest.mark.parametrize("freq,partition_freq", [("1H", "1D"), ("1H", "1w")]) +def test_roundtrip(tmpdir, with_index, freq, partition_freq): dtypes = { "str": object, # FIXME: Categorical data does not work @@ -36,9 +37,8 @@ def test_roundtrip(tmpdir, with_index): ddf = timeseries( start="2023-01-01", end="2023-01-15", - # FIXME: Setting the partition frequency destroys the roundtrip for some - # reason - # partition_freq="1w", + freq=freq, + partition_freq=partition_freq, dtypes=dtypes, ) # FIXME: us is the only precision delta supports. This lib should likely @@ -59,6 +59,7 @@ def test_roundtrip(tmpdir, with_index): if with_index: ddf = ddf.reset_index() + assert ddf.npartitions == ddf_read.npartitions # By default, arrow reads with ns resolution ddf.timestamp = ddf.timestamp.astype("datetime64[ns]") assert_eq(ddf, ddf_read)