IN 1059 - Maintenance 2024-08

Pipfile

@@ -24,6 +24,8 @@ ruff = "*"
 safety = "*"
 pre-commit = "*"
 mypy = "*"
+types-requests = "*"
+boto3-stubs = {extras = ["essential"], version = "*"}
 
 [requires]
 python_version = "3.12"

pyproject.toml

@@ -4,7 +4,8 @@ line-length = 90
 [tool.mypy]
 disallow_untyped_calls = true
 disallow_untyped_defs = true
-exclude = ["tests/"]
+exclude = ["tests/", "output/"]
+ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 log_level = "INFO"

submitter/cli.py

@@ -15,7 +15,7 @@
 
 
 @click.group()
-def main():
+def main() -> None:
     pass
 
 
@@ -24,7 +24,7 @@ def main():
     "--queue", default=CONFIG.INPUT_QUEUE, help="Name of queue to process messages from"
 )
 @click.option("--wait", default=20, help="seconds to wait for long polling. max 20")
-def start(queue, wait):
+def start(queue: str, wait: int) -> None:
     logger.info("Starting processing messages from queue %s", queue)
     message_loop(queue, wait)
     logger.info("Completed processing messages from queue %s", queue)
@@ -50,7 +50,7 @@ def start(queue, wait):
     default="tests/fixtures/completely-fake-data.json",
     help="Path to json file of sample messages to load",
 )
-def load_sample_input_data(input_queue, output_queue, filepath):
+def load_sample_input_data(input_queue: str, output_queue: str, filepath: str) -> None:
     logger.info(f"Loading sample data from file '{filepath}' into queue {input_queue}")
     count = 0
     messages = generate_submission_messages_from_file(filepath, output_queue)
@@ -73,7 +73,7 @@ def load_sample_input_data(input_queue, output_queue, filepath):
     default="tests/fixtures/completely-fake-data.json",
     help="Path to json file of sample messages to load",
 )
-def load_sample_output_data(output_queue, filepath):
+def load_sample_output_data(output_queue: str, filepath: str) -> None:
     logger.info(f"Loading sample data from file '{filepath}' into queue {output_queue}")
     count = 0
     messages = generate_result_messages_from_file(filepath, output_queue)
@@ -85,14 +85,14 @@ def load_sample_output_data(output_queue, filepath):
 
 @main.command()
 @click.argument("name")
-def create_queue(name):
+def create_queue(name: str) -> None:
     """Create queue with NAME supplied as argument"""
     queue = create(name)
     logger.info(queue.url)
 
 
 @main.command()
-def verify_dspace_connection():
+def verify_dspace_connection() -> None:
     client = DSpaceClient(CONFIG.DSPACE_API_URL, timeout=CONFIG.DSPACE_TIMEOUT)
     try:
         client.login(CONFIG.DSPACE_USER, CONFIG.DSPACE_PASSWORD)

submitter/config.py

@@ -5,7 +5,7 @@
 
 
 class Config:
-    def __init__(self):
+    def __init__(self) -> None:
         try:
             self.ENV = os.environ["WORKSPACE"]
         except KeyError as e:
@@ -15,7 +15,21 @@ def __init__(self):
         print(f"Configuring dspace-submission-service for env={self.ENV}")
         self.load_config_variables(self.ENV)
 
-    def load_config_variables(self, env: str):
+    def load_config_variables(self, env: str) -> None:
+        # default to using env vars with defaults
+        self.DSPACE_API_URL = os.getenv("DSPACE_API_URL")
+        self.DSPACE_USER = os.getenv("DSPACE_USER")
+        self.DSPACE_PASSWORD = os.getenv("DSPACE_PASSWORD")
+        self.DSPACE_TIMEOUT = float(os.getenv("DSPACE_TIMEOUT", "120.0"))
+        self.INPUT_QUEUE = os.getenv("INPUT_QUEUE")
+        self.LOG_FILTER = os.getenv("LOG_FILTER", "true").lower()
+        self.LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+        self.SENTRY_DSN = os.getenv("SENTRY_DSN")
+        self.SKIP_PROCESSING = os.getenv("SKIP_PROCESSING", "false").lower()
+        self.SQS_ENDPOINT_URL = os.getenv("SQS_ENDPOINT_URL")
+        self.OUTPUT_QUEUES = os.getenv("OUTPUT_QUEUES", "output").split(",")
+
+        # if testing environment, override
         if env == "test":
             self.DSPACE_API_URL = "mock://dspace.edu/rest/"
             self.DSPACE_USER = "test"
@@ -28,15 +42,3 @@ def load_config_variables(self, env: str):
             self.SKIP_PROCESSING = "false"
             self.SQS_ENDPOINT_URL = "https://sqs.us-east-1.amazonaws.com/"
             self.OUTPUT_QUEUES = ["empty_result_queue"]
-        else:
-            self.DSPACE_API_URL = os.getenv("DSPACE_API_URL")
-            self.DSPACE_USER = os.getenv("DSPACE_USER")
-            self.DSPACE_PASSWORD = os.getenv("DSPACE_PASSWORD")
-            self.DSPACE_TIMEOUT = float(os.getenv("DSPACE_TIMEOUT", "120.0"))
-            self.INPUT_QUEUE = os.getenv("INPUT_QUEUE")
-            self.LOG_FILTER = os.getenv("LOG_FILTER", "true").lower()
-            self.LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
-            self.SENTRY_DSN = os.getenv("SENTRY_DSN")
-            self.SKIP_PROCESSING = os.getenv("SKIP_PROCESSING", "false").lower()
-            self.SQS_ENDPOINT_URL = os.getenv("SQS_ENDPOINT_URL")
-            self.OUTPUT_QUEUES = os.getenv("OUTPUT_QUEUES", "output").split(",")

submitter/errors.py

@@ -57,7 +57,7 @@ class BitstreamAddError(Exception):
         message (str): Explanation of the error
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.message = (
             "Error occurred while parsing bitstream information from files listed in "
             "submission message."

submitter/message.py

@@ -1,7 +1,10 @@
+from collections.abc import Iterator
 import json
 
 
-def generate_submission_messages_from_file(filepath, output_queue):
+def generate_submission_messages_from_file(
+    filepath: str, output_queue: str
+) -> Iterator[tuple[dict, dict]]:
     with open(filepath) as file:
         messages = json.load(file)
 
@@ -11,7 +14,7 @@ def generate_submission_messages_from_file(filepath, output_queue):
         yield attributes, body
 
 
-def attributes_from_json(message_json, output_queue):
+def attributes_from_json(message_json: dict, output_queue: str) -> dict:
     attributes = {
         "PackageID": {
             "DataType": "String",
@@ -29,7 +32,7 @@ def attributes_from_json(message_json, output_queue):
     return attributes
 
 
-def body_from_json(message_json):
+def body_from_json(message_json: dict) -> dict:
     body = {
         "SubmissionSystem": message_json["target system"],
         "CollectionHandle": message_json["collection handle"],
@@ -46,7 +49,9 @@ def body_from_json(message_json):
     return body
 
 
-def generate_result_messages_from_file(filepath, output_queue):
+def generate_result_messages_from_file(
+    filepath: str, _output_queue: str
+) -> Iterator[tuple[dict, dict]]:
     with open(filepath) as file:
         messages = json.load(file)
 
@@ -56,7 +61,7 @@ def generate_result_messages_from_file(filepath, output_queue):
         yield attributes, body
 
 
-def result_attributes_from_json(message_json):
+def result_attributes_from_json(message_json: dict) -> dict:
     attributes = {
         "PackageID": {
             "DataType": "String",
@@ -70,7 +75,7 @@ def result_attributes_from_json(message_json):
     return attributes
 
 
-def result_body_from_json(message_json):
+def result_body_from_json(message_json: dict) -> dict:
     body = {
         "ResultType": message_json["result"],
         "ItemHandle": message_json["handle"],