From 1c2d2e667ac9368fb9046008f39dcad5717d9539 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Tue, 11 Jun 2024 12:28:00 -0400 Subject: [PATCH 1/4] License addition for pyinstaller; experimenting with this in smaht-submitr. --- CHANGELOG.rst | 5 +++++ dcicutils/license_policies/park-lab-common.jsonc | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a43116e3..2e02cf424 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,11 @@ dcicutils Change Log ---------- +8.10.1 +====== +* Changes related to pyinstaller experimentation for smaht-submitr. + + 8.10.0 ====== diff --git a/dcicutils/license_policies/park-lab-common.jsonc b/dcicutils/license_policies/park-lab-common.jsonc index e59d67aee..ae7ad56b2 100644 --- a/dcicutils/license_policies/park-lab-common.jsonc +++ b/dcicutils/license_policies/park-lab-common.jsonc @@ -248,6 +248,12 @@ "docutils" // Used only privately as a separate documentation-generation task for ReadTheDocs ], + + "GNU General Public License v2 (GPLv2)": [ + "pyinstaller", + "pyinstaller-hooks-contrib" + ], + "MIT/X11 Derivative": [ // The license used by libxkbcommon is complicated and involves numerous included licenses, // but all are permissive. diff --git a/pyproject.toml b/pyproject.toml index 4ea9c75b4..59529f6a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.10.0" +version = "8.10.0.1b1" # TODO: To become 8.10.1 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 348d5de867dd2c13e0e1a21e23bc95cd4c261038 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 12 Jun 2024 10:05:32 -0400 Subject: [PATCH 2/4] changes for pyinstaller experimentation --- CHANGELOG.rst | 1 + dcicutils/command_utils.py | 5 +++-- dcicutils/deployment_utils.py | 4 ++-- dcicutils/ecr_scripts.py | 7 ++++--- dcicutils/env_scripts.py | 9 +++++---- dcicutils/progress_bar.py | 2 +- dcicutils/scripts/publish_to_pypi.py | 3 ++- dcicutils/scripts/view_portal_object.py | 14 +++++++------- pyproject.toml | 2 +- test/test_command_utils.py | 3 ++- 10 files changed, 28 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2e02cf424..79ac85da4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Change Log 8.10.1 ====== * Changes related to pyinstaller experimentation for smaht-submitr. + Mostly changing calls to exit to sys.exit; and related license_utils change. 8.10.0 diff --git a/dcicutils/command_utils.py b/dcicutils/command_utils.py index 8b230520a..f98e5e4d6 100644 --- a/dcicutils/command_utils.py +++ b/dcicutils/command_utils.py @@ -7,6 +7,7 @@ import re import requests import subprocess +import sys from typing import Callable, Optional from .exceptions import InvalidParameterError @@ -372,7 +373,7 @@ def fail(*message): raise ScriptFailure(' '.join(message)) try: yield fail - exit(0) + sys.exit(0) except (Exception, ScriptFailure) as e: if DEBUG_SCRIPT: # If debugging, let the error propagate, do not trap it. @@ -384,7 +385,7 @@ def fail(*message): else: message = str(e) # Note: We ignore the type, which isn't intended to be shown. PRINT(message) - exit(1) + sys.exit(1) class Question: diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index b9a919715..8ab95cc8a 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -396,9 +396,9 @@ def main(cls): packaging_was_successful = cls.build_application_version(args.repo, args.version_name, branch=args.branch) if packaging_was_successful: # XXX: how to best detect? time.sleep(5) # give EB a second to catch up (it needs it) - exit(cls.deploy_new_version(args.env, args.repo, args.version_name)) + sys.exit(cls.deploy_new_version(args.env, args.repo, args.version_name)) else: - exit(cls.deploy_indexer(args.env, args.application_version)) + sys.exit(cls.deploy_indexer(args.env, args.application_version)) class IniFileManager: diff --git a/dcicutils/ecr_scripts.py b/dcicutils/ecr_scripts.py index e5d491ce8..d730b7933 100644 --- a/dcicutils/ecr_scripts.py +++ b/dcicutils/ecr_scripts.py @@ -3,6 +3,7 @@ import boto3 import contextlib import os +import sys from typing import Optional, Union, List from .command_utils import yes_or_no @@ -68,7 +69,7 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None): elif account_number != account_number_in_environ: raise RuntimeError("The account number you have specified does not match your declared credentials.") yield ECRCommandContext(account_number=account_number, ecs_repository=ecs_repository, ecr_client=ecr_client) - exit(0) + sys.exit(0) except botocore.exceptions.ClientError as e: error_info = e.response.get('Error', {}) message = error_info.get('Message') @@ -77,12 +78,12 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None): raise RuntimeError("Your security token seems to have expired.") elif message: PRINT(f"{code}: {message}") - exit(1) + sys.exit(1) else: raise except Exception as e: PRINT(f"{full_class_name(e)}: {e}") - exit(1) + sys.exit(1) class ECRCommandContext: diff --git a/dcicutils/env_scripts.py b/dcicutils/env_scripts.py index d38a2ebd1..cf0a96485 100644 --- a/dcicutils/env_scripts.py +++ b/dcicutils/env_scripts.py @@ -1,6 +1,7 @@ import argparse import boto3 import json +import sys import yaml from .lang_utils import disjoined_list @@ -52,7 +53,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): else: PRINT(f"There is no default bucket. Please use a '--bucket' argument" f" to specify one of {disjoined_list(envs_buckets)}.") - exit(1) + sys.exit(1) print_heading(bucket, style='=') @@ -70,7 +71,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): except Exception as e: PRINT("Bucket contents could not be downlaoded.") print_error_message(e) - exit(1) + sys.exit(1) object = None # Without this, PyCharm fusses that object might not get set. -kmp 20-Jul-2022 try: @@ -78,7 +79,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): except Exception as e: PRINT("Bucket contents could not be parsed as JSON.") print_error_message(e) - exit(1) + sys.exit(1) if mode == 'json': PRINT(json.dumps(object, indent=2, default=str)) @@ -86,7 +87,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): PRINT(yaml.dump(object)) else: PRINT(f"Unknown mode: {mode}. Try 'json' or 'yaml'.") - exit(1) + sys.exit(1) DEFAULT_BUCKET = get_env_bucket() diff --git a/dcicutils/progress_bar.py b/dcicutils/progress_bar.py index ac7cfebb3..9862c1a0d 100644 --- a/dcicutils/progress_bar.py +++ b/dcicutils/progress_bar.py @@ -249,7 +249,7 @@ def handle_secondary_interrupt(signum: int, frame: frame) -> None: # noqa if self._interrupt_exit_message: if isinstance(interrupt_exit_message := self._interrupt_exit_message(self), str): print(interrupt_exit_message) - exit(1) + sys.exit(1) elif interrupt_stop is False or ((interrupt_stop is None) and (self._interrupt_exit is False)): set_interrupt_handler(handle_interrupt) interrupt_continue = self._interrupt_continue(self) if self._interrupt_continue else None diff --git a/dcicutils/scripts/publish_to_pypi.py b/dcicutils/scripts/publish_to_pypi.py index a08344996..e89f707f4 100644 --- a/dcicutils/scripts/publish_to_pypi.py +++ b/dcicutils/scripts/publish_to_pypi.py @@ -33,6 +33,7 @@ import os import requests import subprocess +import sys import toml from typing import Tuple, Union @@ -335,7 +336,7 @@ def exit_with_no_action() -> None: first prints a message saying no action was taken. """ PRINT("Exiting without taking action.") - exit(1) + sys.exit(1) PRINT = print diff --git a/dcicutils/scripts/view_portal_object.py b/dcicutils/scripts/view_portal_object.py index 28097bf7b..a6f2369be 100644 --- a/dcicutils/scripts/view_portal_object.py +++ b/dcicutils/scripts/view_portal_object.py @@ -201,17 +201,17 @@ def main(): return else: _print(f"No PATCH data found in file: {args.patch}") - exit(1) + sys.exit(1) data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, database=args.database, check=args.bool, verbose=args.verbose) if args.bool: if data: _print(f"{args.uuid}: found") - exit(0) + sys.exit(0) else: _print(f"{args.uuid}: not found") - exit(1) + sys.exit(1) if args.copy: pyperclip.copy(json.dumps(data, indent=4)) if args.yaml: @@ -597,18 +597,18 @@ def tree_generator(name: str, prefix: str = ""): def _read_json_from_file(file: str) -> Optional[dict]: if not os.path.exists(file): _print(f"Cannot find file: {file}") - exit(1) + sys.exit(1) try: with io.open(file, "r") as f: try: return json.load(f) except Exception: _print(f"Cannot parse JSON in file: {file}") - exit(1) + sys.exit(1) except Exception as e: print(e) _print(f"Cannot open file: {file}") - exit(1) + sys.exit(1) def _print(*args, **kwargs): @@ -620,7 +620,7 @@ def _print(*args, **kwargs): def _exit(message: Optional[str] = None) -> None: if message: _print(f"ERROR: {message}") - exit(1) + sys.exit(1) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 59529f6a7..776dbab77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.10.0.1b1" # TODO: To become 8.10.1 +version = "8.10.0.1b2" # TODO: To become 8.10.1 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/test/test_command_utils.py b/test/test_command_utils.py index 853006030..358d58d6c 100644 --- a/test/test_command_utils.py +++ b/test/test_command_utils.py @@ -1,5 +1,6 @@ import os import pytest +import sys import tempfile from unittest import mock @@ -378,7 +379,7 @@ def test_script_catch_errors(): with script_catch_errors(): PRINT(normal_output) PRINT(custom_exit_message) - exit(1) # Bypasses script_catch_errors context manager, so won't show SCRIPT_ERROR_HERALD + sys.exit(1) # Bypasses script_catch_errors context manager, so won't show SCRIPT_ERROR_HERALD sys_exit = exit_exc.value assert isinstance(sys_exit, SystemExit) assert sys_exit.code == 1 From fcbfca597a76c2c885d2db2f682089b775932008 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 12 Jun 2024 15:10:02 -0400 Subject: [PATCH 3/4] Added hook to structured_data (row_reader_hook) for integration testing purposes. --- CHANGELOG.rst | 1 + dcicutils/structured_data.py | 4 ++++ pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 79ac85da4..09c73c29f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Change Log ====== * Changes related to pyinstaller experimentation for smaht-submitr. Mostly changing calls to exit to sys.exit; and related license_utils change. +* Added hook to structured_data (row_reader_hook) for integration testing purposes. 8.10.0 diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py index c9b895849..2d058a588 100644 --- a/dcicutils/structured_data.py +++ b/dcicutils/structured_data.py @@ -57,6 +57,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp ref_lookup_nocache: bool = False, norefs: bool = False, merge: bool = False, progress: Optional[Callable] = None, + row_reader_hook: Optional[Callable] = None, debug_sleep: Optional[str] = None) -> None: self._progress = progress if callable(progress) else None self._data = {} @@ -75,6 +76,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None self._norefs = True if norefs is True else False self._merge = True if merge is True else False # New merge functionality (2024-05-25) + self._row_reader_hook = row_reader_hook if callable(row_reader_hook) else None # Testing support (2024-06-12) self._debug_sleep = None if debug_sleep: try: @@ -377,6 +379,8 @@ def _load_reader(self, reader: RowReader, type_name: str) -> None: structured_row_template = _StructuredRowTemplate(reader.header, schema) structured_row = structured_row_template.create_row() for column_name, value in row.items(): + if self._row_reader_hook: + value = self._row_reader_hook(reader.sheet_name, column_name, value) structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number) if self._autoadd_properties: self._add_properties(structured_row, self._autoadd_properties, schema) diff --git a/pyproject.toml b/pyproject.toml index 776dbab77..5d5d86100 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.10.0.1b2" # TODO: To become 8.10.1 +version = "8.10.0.1b3" # TODO: To become 8.10.1 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 7bef200f5ed98717e63d9872c560b55778a8d335 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Mon, 24 Jun 2024 13:55:25 -0400 Subject: [PATCH 4/4] update version to 8.12.0 - ready to merge pr-310 to master --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ae3bd7e21..783e76e47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.11.0.1b1" # TODO: To become 8.12.0 +version = "8.12.0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT"