From 2539c5ff126e4544935a4e11e7fdf6ccaa83468f Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 22 Aug 2024 07:14:20 -0400 Subject: [PATCH 1/6] Added portal_utils.Portal.head method --- CHANGELOG.rst | 7 +++++++ dcicutils/portal_utils.py | 11 +++++++++++ pyproject.toml | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f867ea7b6..6664c0284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,13 @@ dcicutils Change Log ---------- +8.14.3 +====== + +* 2024-08-22 (dmichaels) +* Added portal_utils.Portal.head method. + + 8.14.2 ====== * Corrected requests version (to ^2.27.0 from 2.31.0) on pyproject.toml to not be pinned; but doing diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py index 0f0bba5e8..c73d565b6 100644 --- a/dcicutils/portal_utils.py +++ b/dcicutils/portal_utils.py @@ -280,6 +280,17 @@ def post_metadata(self, object_type: str, data: dict, check_only: bool = False) add_on="check_only=True" if check_only else "") return self.post(f"/{object_type}{'?check_only=True' if check_only else ''}", data).json() + def head(self, url: str, follow: bool = False, raise_exception: bool = False, **kwargs) -> Optional[int]: + try: + response = requests.head(self.url(url), **self._kwargs(**kwargs)) + if response and response.status_code in [301, 302, 303, 307, 308] and (follow is True): + response = response.follow() + return response.status_code + except Exception as e: + if raise_exception is True: + raise e + return None + def get_health(self) -> OptionalResponse: return self.get("/health") diff --git a/pyproject.toml b/pyproject.toml index 630a224fd..33e115f58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.14.2" +version = "8.14.2.1b1" # TODO: To become 8.14.3 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 29256a24347dda0b1986629af90e97e966ccdd3c Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 22 Aug 2024 07:33:01 -0400 Subject: [PATCH 2/6] Added portal_utils.Portal.head method --- dcicutils/portal_utils.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py index c73d565b6..ce92c9270 100644 --- a/dcicutils/portal_utils.py +++ b/dcicutils/portal_utils.py @@ -280,10 +280,10 @@ def post_metadata(self, object_type: str, data: dict, check_only: bool = False) add_on="check_only=True" if check_only else "") return self.post(f"/{object_type}{'?check_only=True' if check_only else ''}", data).json() - def head(self, url: str, follow: bool = False, raise_exception: bool = False, **kwargs) -> Optional[int]: + def head(self, url: str, follow: bool = True, raise_exception: bool = False, **kwargs) -> Optional[int]: try: response = requests.head(self.url(url), **self._kwargs(**kwargs)) - if response and response.status_code in [301, 302, 303, 307, 308] and (follow is True): + if response and response.status_code in [301, 302, 303, 307, 308] and (follow is not False): response = response.follow() return response.status_code except Exception as e: diff --git a/pyproject.toml b/pyproject.toml index 33e115f58..d418e80e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.14.2.1b1" # TODO: To become 8.14.3 +version = "8.14.2.1b2" # TODO: To become 8.14.3 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 2b3eaa0ce73af4c225d16e9be4eeba4e9723d9dc Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 22 Aug 2024 18:47:43 -0400 Subject: [PATCH 3/6] Modified structured_data hook interface slightly to check for and call the "finish" attribute/callable hook. --- CHANGELOG.rst | 2 ++ dcicutils/structured_data.py | 4 ++++ pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6664c0284..f3e8e651c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Change Log ====== * 2024-08-22 (dmichaels) +* Modified structured_data hook interface slightly to + check for and call the "finish" attribute/callable hook. * Added portal_utils.Portal.head method. diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py index 8bb134432..74733061f 100644 --- a/dcicutils/structured_data.py +++ b/dcicutils/structured_data.py @@ -287,6 +287,10 @@ def _load_normal_file(self, file: str) -> None: self._load_json_file(file) elif file.endswith(".tar") or file.endswith(".zip"): self._load_packed_file(file) + if (self._validator_hook and + hasattr(self._validator_hook, "finish") and + callable(finish_validator_hook := getattr(self._validator_hook, "finish"))): # noqa + finish_validator_hook(self) def _load_packed_file(self, file: str) -> None: for file in unpack_files(file, suffixes=ACCEPTABLE_FILE_SUFFIXES): diff --git a/pyproject.toml b/pyproject.toml index d418e80e6..e9ceffc66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.14.2.1b2" # TODO: To become 8.14.3 +version = "8.14.2.1b3" # TODO: To become 8.14.3 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 159e5ae742ef640d0104673d1cebc154c5982f73 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 22 Aug 2024 23:52:45 -0400 Subject: [PATCH 4/6] Modified structured_data hook interface slightly to check for and call the "finish" attribute/callable hook. --- CHANGELOG.rst | 4 ++-- dcicutils/structured_data.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3e8e651c..6926218df 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,8 +10,8 @@ Change Log ====== * 2024-08-22 (dmichaels) -* Modified structured_data hook interface slightly to - check for and call the "finish" attribute/callable hook. +* Modified structured_data property hook for "finish" attribute/callable. +* Added sheet hook to structured_data. * Added portal_utils.Portal.head method. diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py index 74733061f..945723040 100644 --- a/dcicutils/structured_data.py +++ b/dcicutils/structured_data.py @@ -58,6 +58,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp norefs: bool = False, merge: bool = False, progress: Optional[Callable] = None, validator_hook: Optional[Callable] = None, + validator_sheet_hook: Optional[Callable] = None, debug_sleep: Optional[str] = None) -> None: self._progress = progress if callable(progress) else None self._data = {} @@ -76,7 +77,8 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None self._norefs = True if norefs is True else False self._merge = True if merge is True else False # New merge functionality (2024-05-25) - self._validator_hook = validator_hook if callable(validator_hook) else None # Testing support (2024-06-12) + self._validator_hook = validator_hook if callable(validator_hook) else None + self._validator_sheet_hook = validator_sheet_hook if callable(validator_sheet_hook) else None self._debug_sleep = None if debug_sleep: try: @@ -316,6 +318,8 @@ def get_counts() -> Tuple[int, int]: order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {} for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)): self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name)) + if self._validator_sheet_hook: + self._validator_sheet_hook(self, sheet_name, self.data[sheet_name]) # TODO: Do we really need progress reporting for the below? # Check for unresolved reference errors which really are not because of ordering. # Yes such internal references will be handled correctly on actual database update via snovault.loadxl. From c5efa11bf4ee87f0fac38d4d49affd2096c6baeb Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 22 Aug 2024 23:52:51 -0400 Subject: [PATCH 5/6] Modified structured_data hook interface slightly to check for and call the "finish" attribute/callable hook. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e9ceffc66..e6c9c393e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.14.2.1b3" # TODO: To become 8.14.3 +version = "8.14.2.1b4" # TODO: To become 8.14.3 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 734bba9f0f10b1d07a94f47b9103218231d809cc Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 23 Aug 2024 10:28:28 -0400 Subject: [PATCH 6/6] version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6c9c393e..3d1dd5044 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.14.2.1b4" # TODO: To become 8.14.3 +version = "8.14.3" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT"