From 085babd1dcf88cc499a75c2f5d86553571fbefc0 Mon Sep 17 00:00:00 2001 From: Marcus Read Date: Fri, 2 Feb 2024 11:57:40 +0000 Subject: [PATCH] Fix for pandas pre 2.2.0 Fixes to accommodate pandas pre 2.2.0 default frequency strings, "T" (now "min"), "H" (now "h"), "S" (now "s"), "M" (now "ME") and "Y" (now "YE"). --- .github/workflows/release.yml | 1 + src/market_prices/intervals.py | 11 +++++++--- src/market_prices/pt.py | 10 ++++++++- src/market_prices/utils/pandas_utils.py | 28 +++++++++++++++++++++++++ tests/conftest.py | 13 ++++++++++++ tests/hypstrtgy.py | 15 ++++++++++++- tests/test_pt.py | 6 ++++-- 7 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee26962..6ceccb5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,6 +37,7 @@ jobs: - name: Install from testpypi and import shell: bash run: | + sleep 5 i=0 while [ $i -lt 12 ] && [ "${{ github.ref_name }}" != $(pip index versions -i https://test.pypi.org/simple --pre market_prices | cut -d'(' -f2 | cut -d')' -f1 | sed 1q) ];\ do echo "waiting for package to appear in test index, i is $i"; echo "sleeping 5s"; sleep 5s; echo "woken up"; let i++; echo "next i is $i"; done diff --git a/src/market_prices/intervals.py b/src/market_prices/intervals.py index 31fbccf..b039382 100644 --- a/src/market_prices/intervals.py +++ b/src/market_prices/intervals.py @@ -56,7 +56,13 @@ def freq_unit(self) -> typing.Literal["min", "h", "D"]: Returns either "min", "h" or "D". """ - return self.as_pdtd.resolution_string + unit = self.as_pdtd.resolution_string + # for pre pandas 2.2 compatibility... + if unit == "T": + unit = "min" + if unit == "H": + unit = "h" + return unit @property def freq_value(self) -> int: @@ -449,8 +455,7 @@ def to_ptinterval(interval: str | timedelta | pd.Timedelta) -> PTInterval: " interval in terms of months pass as a string, for" ' example "1m" for one month.' ) - - valid_resolutions = ["min", "h", "D"] + valid_resolutions = ["min", "h", "D"] + ["T", "H"] # + form pandas pre 2.2 if interval.resolution_string not in valid_resolutions: raise ValueError(error_msg) diff --git a/src/market_prices/pt.py b/src/market_prices/pt.py index 0884292..e057d76 100644 --- a/src/market_prices/pt.py +++ b/src/market_prices/pt.py @@ -1480,7 +1480,9 @@ def downsample( # pylint: disable=arguments-differ else: return self._downsample_days(pdfreq) - if unit in ["h", "min", "s", "L", "ms", "U", "us", "N", "ns"]: + invalid_units = ["h", "min", "MIN", "s", "L", "ms", "U", "us", "N", "ns"] + ext = ["t", "T", "H", "S"] # for pandas pre 2.2 compatibility + if unit in invalid_units + ext: raise ValueError( "Cannot downsample to a `pdfreq` with a unit more precise than 'd'." ) @@ -2328,6 +2330,12 @@ def downsample( ) unit = genutils.remove_digits(pdfreq) + # for pandas pre 2.2. compatibility + if unit == "T": + unit = "min" + if unit == "H": + unit = "h" + valid_units = ["min", "h"] if unit not in valid_units: raise ValueError( diff --git a/src/market_prices/utils/pandas_utils.py b/src/market_prices/utils/pandas_utils.py index 9b582a1..0c86135 100644 --- a/src/market_prices/utils/pandas_utils.py +++ b/src/market_prices/utils/pandas_utils.py @@ -76,6 +76,19 @@ def timestamps_in_interval_of_intervals( Examples -------- + >>> # ignore first part, for testing purposes only... + >>> import pytest, pandas + >>> v = pandas.__version__ + >>> if ( + ... (v.count(".") == 1 and float(v) < 2.2) + ... or ( + ... v.count(".") > 1 + ... and float(v[:v.index(".", v.index(".") + 1)]) < 2.2 + ... ) + ... ): + ... pytest.skip("printed return only valid from pandas 2.2") + >>> # + >>> # example from here... >>> timestamps = pd.DatetimeIndex( ... [ ... pd.Timestamp('2021-03-12 14:00'), @@ -96,6 +109,7 @@ def timestamps_in_interval_of_intervals( >>> timestamps_in_interval_of_intervals(timestamps, intervals) True """ + # NOTE Can lose doctest skip when pandas support is >= 2.2 timestamps = [timestamps] if isinstance(timestamps, pd.Timestamp) else timestamps ser = intervals.to_series() bv = ser.apply(lambda x: all({ts in x for ts in timestamps})) @@ -387,6 +401,19 @@ def remove_intervals_from_interval( Examples -------- + >>> # ignore first part, for testing purposes only... + >>> import pytest, pandas + >>> v = pandas.__version__ + >>> if ( + ... (v.count(".") == 1 and float(v) < 2.2) + ... or ( + ... v.count(".") > 1 + ... and float(v[:v.index(".", v.index(".") + 1)]) < 2.2 + ... ) + ... ): + ... pytest.skip("printed return only valid from pandas 2.2") + >>> # + >>> # example from here... >>> from pprint import pprint >>> left = pd.date_range('2021-05-01 12:00', periods=5, freq='h') >>> right = left + pd.Timedelta(30, 'min') @@ -411,6 +438,7 @@ def remove_intervals_from_interval( Interval(2021-05-01 15:30:00, 2021-05-01 16:00:00, closed='left'), Interval(2021-05-01 16:30:00, 2021-05-01 17:30:00, closed='left')] """ + # NOTE Can lose doctest skip when pandas support is >= 2.2 if not intervals.is_monotonic_increasing: raise ValueError( "`intervals` must be monotonically increasing although receieved" diff --git a/tests/conftest.py b/tests/conftest.py index c6a0cbf..88ff8f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -280,3 +280,16 @@ def xlon_calendar_extended( def xhkg_calendar(today, side, mock_now) -> abc.Iterator[xcals.ExchangeCalendar]: """XLON calendar.""" yield xcals.get_calendar("XHKG", side=side, end=today) + + +@pytest.fixture +def pandas_pre_22() -> abc.Iterator[bool]: + """Installed pandas is pre version 2.2.""" + v = pd.__version__ + if v.count(".") == 1: + rtrn = float(v) < 2.2 + else: + stop = v.index(".", v.index(".") + 1) + minor_v = float(v[:stop]) + rtrn = minor_v < 2.2 + yield rtrn diff --git a/tests/hypstrtgy.py b/tests/hypstrtgy.py index f68e3d6..c8aef28 100644 --- a/tests/hypstrtgy.py +++ b/tests/hypstrtgy.py @@ -361,6 +361,16 @@ def pp_days(draw, calendar_name: str) -> st.SearchStrategy[dict[str, typing.Any] return pp +# set PRE_PANDAS_22 +v = pd.__version__ +if v.count(".") == 1: + PRE_PANDAS_22 = float(v) < 2.2 +else: + stop = v.index(".", v.index(".") + 1) + minor_v = float(v[:stop]) + PRE_PANDAS_22 = minor_v < 2.2 + + @st.composite def pp_days_start_session( draw, @@ -387,7 +397,10 @@ def pp_days_start_session( sessions = calendar.sessions limit_r = sessions[-pp["days"]] if start_will_roll_to_ms: - offset = pd.tseries.frequencies.to_offset("ME") + # NOTE when min pandas support moves to >= 2.2 can hard code this as ME + # and lose the PRE_PANDAS_22 global. + freq = "M" if PRE_PANDAS_22 else "ME" + offset = pd.tseries.frequencies.to_offset(freq) if TYPE_CHECKING: assert offset is not None limit_r = offset.rollback(limit_r) diff --git a/tests/test_pt.py b/tests/test_pt.py index 69288b1..620897d 100644 --- a/tests/test_pt.py +++ b/tests/test_pt.py @@ -2289,7 +2289,7 @@ def xnys_open(self, xnys, session) -> abc.Iterator[pd.Timestamp]: def xnys_close(self, xnys, session) -> abc.Iterator[pd.Timestamp]: yield xnys.session_close(session) - def test_errors(self, intraday_pt, composite_intraday_pt, one_min): + def test_errors(self, intraday_pt, composite_intraday_pt, one_min, pandas_pre_22): """Verify raising expected errors for intraday price table.""" df = intraday_pt f = df.pt.downsample @@ -2328,7 +2328,9 @@ def match_f(pdfreq) -> str: f" received `pdfreq` as {pdfreq}." ) - invalid_pdfreqs = ["1d", "1s", "1ns", "1ms", "1ME", "1YE"] + invalid_pdfreqs = ["1d", "1s", "1ns", "1ms"] + ext = ["1M", "1Y"] if pandas_pre_22 else ["1ME", "1YE"] + invalid_pdfreqs += ext for pdfreq in invalid_pdfreqs: with pytest.raises(ValueError, match=match_f(pdfreq)): f(pdfreq)