Merge pull request #310 from 4dn-dcic/pyinstaller-experiment-20240611

License addition for pyinstaller; experimenting with this in submitr

CHANGELOG.rst

@@ -6,6 +6,13 @@ dcicutils
 Change Log
 ----------
 
+8.12.0
+======
+* Changes related to pyinstaller experimentation for smaht-submitr.
+  Mostly changing calls to exit to sys.exit; and related license_utils change.
+* Added hook to structured_data (row_reader_hook) for integration testing purposes.
+
+
 8.11.0
 ======
 

dcicutils/command_utils.py

@@ -7,6 +7,7 @@
 import re
 import requests
 import subprocess
+import sys
 
 from typing import Callable, Optional
 from .exceptions import InvalidParameterError
@@ -372,7 +373,7 @@ def fail(*message):
         raise ScriptFailure(' '.join(message))
     try:
         yield fail
-        exit(0)
+        sys.exit(0)
     except (Exception, ScriptFailure) as e:
         if DEBUG_SCRIPT:
             # If debugging, let the error propagate, do not trap it.
@@ -384,7 +385,7 @@ def fail(*message):
             else:
                 message = str(e)  # Note: We ignore the type, which isn't intended to be shown.
                 PRINT(message)
-            exit(1)
+            sys.exit(1)
 
 
 class Question:

dcicutils/deployment_utils.py

@@ -396,9 +396,9 @@ def main(cls):
             packaging_was_successful = cls.build_application_version(args.repo, args.version_name, branch=args.branch)
             if packaging_was_successful:  # XXX: how to best detect?
                 time.sleep(5)  # give EB a second to catch up (it needs it)
-                exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
+                sys.exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
         else:
-            exit(cls.deploy_indexer(args.env, args.application_version))
+            sys.exit(cls.deploy_indexer(args.env, args.application_version))
 
 
 class IniFileManager:

dcicutils/ecr_scripts.py

@@ -3,6 +3,7 @@
 import boto3
 import contextlib
 import os
+import sys
 
 from typing import Optional, Union, List
 from .command_utils import yes_or_no
@@ -68,7 +69,7 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
             elif account_number != account_number_in_environ:
                 raise RuntimeError("The account number you have specified does not match your declared credentials.")
             yield ECRCommandContext(account_number=account_number, ecs_repository=ecs_repository, ecr_client=ecr_client)
-            exit(0)
+            sys.exit(0)
         except botocore.exceptions.ClientError as e:
             error_info = e.response.get('Error', {})
             message = error_info.get('Message')
@@ -77,12 +78,12 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
                 raise RuntimeError("Your security token seems to have expired.")
             elif message:
                 PRINT(f"{code}: {message}")
-                exit(1)
+                sys.exit(1)
             else:
                 raise
     except Exception as e:
         PRINT(f"{full_class_name(e)}: {e}")
-        exit(1)
+        sys.exit(1)
 
 
 class ECRCommandContext:

dcicutils/env_scripts.py

@@ -1,6 +1,7 @@
 import argparse
 import boto3
 import json
+import sys
 import yaml
 
 from .lang_utils import disjoined_list
@@ -52,7 +53,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         else:
             PRINT(f"There is no default bucket. Please use a '--bucket' argument"
                   f" to specify one of {disjoined_list(envs_buckets)}.")
-        exit(1)
+        sys.exit(1)
 
     print_heading(bucket, style='=')
 
@@ -70,23 +71,23 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         except Exception as e:
             PRINT("Bucket contents could not be downlaoded.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         object = None  # Without this, PyCharm fusses that object might not get set. -kmp 20-Jul-2022
         try:
             object = json.load(object_stream)
         except Exception as e:
             PRINT("Bucket contents could not be parsed as JSON.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         if mode == 'json':
             PRINT(json.dumps(object, indent=2, default=str))
         elif mode == 'yaml':
             PRINT(yaml.dump(object))
         else:
             PRINT(f"Unknown mode: {mode}. Try 'json' or 'yaml'.")
-            exit(1)
+            sys.exit(1)
 
 
 DEFAULT_BUCKET = get_env_bucket()

dcicutils/license_policies/park-lab-common.jsonc

@@ -248,6 +248,12 @@
             "docutils"  // Used only privately as a separate documentation-generation task for ReadTheDocs
         ],
 
+
+        "GNU General Public License v2 (GPLv2)": [
+            "pyinstaller",
+            "pyinstaller-hooks-contrib"
+        ],
+
         "MIT/X11 Derivative": [
             // The license used by libxkbcommon is complicated and involves numerous included licenses,
             // but all are permissive.

dcicutils/progress_bar.py

@@ -249,7 +249,7 @@ def handle_secondary_interrupt(signum: int, frame: frame) -> None:  # noqa
                         if self._interrupt_exit_message:
                             if isinstance(interrupt_exit_message := self._interrupt_exit_message(self), str):
                                 print(interrupt_exit_message)
-                        exit(1)
+                        sys.exit(1)
                     elif interrupt_stop is False or ((interrupt_stop is None) and (self._interrupt_exit is False)):
                         set_interrupt_handler(handle_interrupt)
                         interrupt_continue = self._interrupt_continue(self) if self._interrupt_continue else None

dcicutils/scripts/publish_to_pypi.py

@@ -33,6 +33,7 @@
 import os
 import requests
 import subprocess
+import sys
 import toml
 
 from typing import Tuple, Union
@@ -335,7 +336,7 @@ def exit_with_no_action() -> None:
     first prints a message saying no action was taken.
     """
     PRINT("Exiting without taking action.")
-    exit(1)
+    sys.exit(1)
 
 
 PRINT = print

dcicutils/scripts/view_portal_object.py

@@ -201,17 +201,17 @@ def main():
             return
         else:
             _print(f"No PATCH data found in file: {args.patch}")
-            exit(1)
+            sys.exit(1)
 
     data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw,
                               database=args.database, check=args.bool, verbose=args.verbose)
     if args.bool:
         if data:
             _print(f"{args.uuid}: found")
-            exit(0)
+            sys.exit(0)
         else:
             _print(f"{args.uuid}: not found")
-            exit(1)
+            sys.exit(1)
     if args.copy:
         pyperclip.copy(json.dumps(data, indent=4))
     if args.yaml:
@@ -597,18 +597,18 @@ def tree_generator(name: str, prefix: str = ""):
 def _read_json_from_file(file: str) -> Optional[dict]:
     if not os.path.exists(file):
         _print(f"Cannot find file: {file}")
-        exit(1)
+        sys.exit(1)
     try:
         with io.open(file, "r") as f:
             try:
                 return json.load(f)
             except Exception:
                 _print(f"Cannot parse JSON in file: {file}")
-                exit(1)
+                sys.exit(1)
     except Exception as e:
         print(e)
         _print(f"Cannot open file: {file}")
-        exit(1)
+        sys.exit(1)
 
 
 def _print(*args, **kwargs):
@@ -620,7 +620,7 @@ def _print(*args, **kwargs):
 def _exit(message: Optional[str] = None) -> None:
     if message:
         _print(f"ERROR: {message}")
-    exit(1)
+    sys.exit(1)
 
 
 if __name__ == "__main__":

dcicutils/structured_data.py

@@ -57,6 +57,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp
                  ref_lookup_nocache: bool = False,
                  norefs: bool = False, merge: bool = False,
                  progress: Optional[Callable] = None,
+                 row_reader_hook: Optional[Callable] = None,
                  debug_sleep: Optional[str] = None) -> None:
         self._progress = progress if callable(progress) else None
         self._data = {}
@@ -75,6 +76,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp
         self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._norefs = True if norefs is True else False
         self._merge = True if merge is True else False  # New merge functionality (2024-05-25)
+        self._row_reader_hook = row_reader_hook if callable(row_reader_hook) else None  # Testing support (2024-06-12)
         self._debug_sleep = None
         if debug_sleep:
             try:
@@ -377,6 +379,8 @@ def _load_reader(self, reader: RowReader, type_name: str) -> None:
                 structured_row_template = _StructuredRowTemplate(reader.header, schema)
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
+                if self._row_reader_hook:
+                    value = self._row_reader_hook(reader.sheet_name, column_name, value)
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
                 if self._autoadd_properties:
                     self._add_properties(structured_row, self._autoadd_properties, schema)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.11.0"
+version = "8.12.0"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

test/test_command_utils.py

@@ -1,5 +1,6 @@
 import os
 import pytest
+import sys
 import tempfile
 
 from unittest import mock
@@ -378,7 +379,7 @@ def test_script_catch_errors():
             with script_catch_errors():
                 PRINT(normal_output)
                 PRINT(custom_exit_message)
-                exit(1)  # Bypasses script_catch_errors context manager, so won't show SCRIPT_ERROR_HERALD
+                sys.exit(1)  # Bypasses script_catch_errors context manager, so won't show SCRIPT_ERROR_HERALD
         sys_exit = exit_exc.value
         assert isinstance(sys_exit, SystemExit)
         assert sys_exit.code == 1