From e8246f95c0e51b3f8ba3593bee676cbe205b27ae Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 18 May 2023 13:34:32 -0300 Subject: [PATCH 1/3] remove MemoizedDataset --- compliance_checker/__init__.py | 16 ---------------- compliance_checker/base.py | 4 ++-- compliance_checker/cf/cf_1_8.py | 3 +-- compliance_checker/suite.py | 8 ++++---- pyproject.toml | 1 - 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/compliance_checker/__init__.py b/compliance_checker/__init__.py index 783dd071..f5975758 100644 --- a/compliance_checker/__init__.py +++ b/compliance_checker/__init__.py @@ -1,29 +1,13 @@ from contextlib import contextmanager -from functools import lru_cache from tempfile import NamedTemporaryFile from typing import BinaryIO, Generator -from netCDF4 import Dataset - try: from ._version import __version__ except ImportError: __version__ = "unknown" -class MemoizedDataset(Dataset): - """ - A NetCDF dataset which has its get_variables_by_attributes call memoized in - order to speed up repeated calls to the function. This should only really - be used against netCDF Datasets opened in 'r' mode, as the attributes should - not change upon reading the files. - """ - - @lru_cache(128) - def get_variables_by_attributes(self, **kwargs): - return super().get_variables_by_attributes(**kwargs) - - @contextmanager def tempnc(data: BinaryIO) -> Generator[str, None, None]: """ diff --git a/compliance_checker/base.py b/compliance_checker/base.py index 34981221..347bee7d 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -20,7 +20,7 @@ from owslib.swe.sensor.sml import SensorML import compliance_checker.cfutil as cfutil -from compliance_checker import MemoizedDataset, __version__ +from compliance_checker import __version__ from compliance_checker.util import kvp_convert # Python 3.5+ should work, also have a fallback @@ -201,7 +201,7 @@ class BaseNCCheck: Base Class for NetCDF Dataset supporting Check Suites. """ - supported_ds = {Dataset, MemoizedDataset} + supported_ds = [Dataset] @classmethod def std_check_in(cls, dataset, name, allowed_vals): diff --git a/compliance_checker/cf/cf_1_8.py b/compliance_checker/cf/cf_1_8.py index 3c1f5d79..0f001a8c 100644 --- a/compliance_checker/cf/cf_1_8.py +++ b/compliance_checker/cf/cf_1_8.py @@ -20,7 +20,6 @@ from netCDF4 import Dataset from shapely.geometry import Polygon -from compliance_checker import MemoizedDataset from compliance_checker.base import BaseCheck, TestCtx from compliance_checker.cf.cf_1_7 import CF1_7Check from compliance_checker.cf.util import reference_attr_variables, string_from_var_type @@ -46,7 +45,7 @@ def __init__(self, options=None): }, ) - def check_groups(self, ds: MemoizedDataset): + def check_groups(self, ds: Dataset): """ 2.7.2. Application of attributes diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index 3eb6ecad..7f435208 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -25,7 +25,7 @@ from owslib.swe.sensor.sml import SensorML from pkg_resources import working_set -from compliance_checker import MemoizedDataset, __version__, tempnc +from compliance_checker import __version__, tempnc from compliance_checker.base import BaseCheck, GenericFile, Result, fix_return_value from compliance_checker.protocols import cdl, netcdf, opendap @@ -828,7 +828,7 @@ def check_remote_netcdf(self, ds_str): if netcdf.is_remote_netcdf(ds_str): response = requests.get(ds_str, allow_redirects=True, timeout=60) try: - return MemoizedDataset( + return Dataset( urlparse(response.url).path, memory=response.content, ) @@ -836,7 +836,7 @@ def check_remote_netcdf(self, ds_str): # handle case when netCDF C libs weren't compiled with # in-memory support by using tempfile with tempnc(response.content) as _nc: - return MemoizedDataset(_nc) + return Dataset(_nc) def load_remote_dataset(self, ds_str): """ @@ -893,7 +893,7 @@ def load_local_dataset(self, ds_str): ds_str = self.generate_dataset(ds_str) if netcdf.is_netcdf(ds_str): - return MemoizedDataset(ds_str) + return Dataset(ds_str) # Assume this is just a Generic File if it exists if os.path.isfile(ds_str): diff --git a/pyproject.toml b/pyproject.toml index 63198262..2cd038aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,5 +39,4 @@ ignore = [ "E402", "A001", ] -"compliance_checker/__init__.py" = ["B019"] "compliance_checker/cfutil.py" = ["B028"] From 29378c4b64dbc50bbf1c2ca45e753e1b33cceff8 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 6 Jun 2023 11:35:13 -0300 Subject: [PATCH 2/3] restrict to the upstream feat ver --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47b9921c..3df7b0d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cftime>=1.1.0 isodate>=0.6.1 jinja2>=2.7.3 lxml>=3.2.1 -netcdf4>=1.5.7 +netcdf4>=1.6.4 owsLib>=0.8.3 pendulum>=1.2.4 pygeoif>=0.6 From 4d66c02fc04d751688559f5a08fccc3e7d7a65f0 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 6 Jun 2023 11:35:25 -0300 Subject: [PATCH 3/3] update min python --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 00b4877c..e26a432b 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def pip_requirements(fname="requirements.txt"): url="https://github.com/ioos/compliance-checker", packages=find_packages(), install_requires=pip_requirements(), - python_requires="~=3.5", + python_requires="~=3.7", tests_require=["pytest"], classifiers=[ "Development Status :: 5 - Production/Stable",