Log unhandled exceptions for debugging

Why these changes are being introduced:

Sentry recently reported, 'Unexpected exception, aborting DSpace
Submission Service processing', which is a prepared logging statement as
part of Submission.submit().  This suggests that a totally unhandled
exception was encountered during submit work.  While somewhat helpful,
this does not give us insight into the underlying exception that caused
it in Sentry or Cloudwatch.

How this addresses that need:
* Changes logger.error to logger.exception to log the underlying
Exception to Sentry with a full traceback
* Though we may still need work to rollback actions taken,
or manage the SQS queue based on an unhandled exception,
this change will provide more information in these situations

Side effects of this change:
* Unhandled exceptions will be fully logged to Sentry

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/IN-1035

submitter/submission.py

@@ -188,7 +188,7 @@ def submit(self, client):
             self.result_error_message(e.message, getattr(e, "dspace_error", None))
             clean_up_partial_success(client, item)
         except Exception as e:
-            logger.error(
+            logger.exception(
                 "Unexpected exception, aborting DSpace Submission Service processing"
             )
             raise e

tests/conftest.py

@@ -52,6 +52,15 @@ def test_aws_user(aws_credentials):
 
 @pytest.fixture(scope="function")
 def mocked_dspace():
+    """The following mock responses from DSpace based on the URL of the request.
+
+    Fixtures below that prepare an SQS message, where specific collections or bitstreams
+    are included, will utilize these mocked responses from DSpace.
+
+    EXAMPLE: fixture 'input_message_item_post_dspace_timeout' sets collection
+    "CollectionHandle: 0000/collection03".  This aligns with a URL defined here, and will
+    therefore throw a requests.exceptions.ConnectTimeout exception to test against.
+    """
     with requests_mock.Mocker() as m:
         m.post(
             "mock://dspace.edu/rest/login",
@@ -69,6 +78,12 @@ def mocked_dspace():
             "mock://dspace.edu/rest/handle/0000/collection03",
             exc=exceptions.ConnectTimeout,
         )
+        m.get(
+            "mock://dspace.edu/rest/handle/0000/collection04",
+            exc=exceptions.RequestException(
+                "Catastrophic error before or during request!  No response to parse."
+            ),
+        )
         m.get(
             "mock://dspace.edu/rest/handle/0000/not-a-collection",
             status_code=404,
@@ -347,6 +362,30 @@ def input_message_bitstream_file_open_error(mocked_sqs):
     yield message
 
 
+@pytest.fixture
+def input_message_item_post_dspace_generic_500_error(mocked_sqs):
+    queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue")
+    queue.send_message(
+        MessageAttributes=test_attributes,
+        MessageBody=json.dumps(
+            {
+                "SubmissionSystem": "DSpace@MIT",
+                "CollectionHandle": "0000/collection04",
+                "MetadataLocation": "tests/fixtures/test-item-metadata.json",
+                "Files": [
+                    {
+                        "BitstreamName": "test-file-01.pdf",
+                        "FileLocation": "tests/fixtures/test-file-01.pdf",
+                        "BitstreamDescription": "A test bitstream",
+                    }
+                ],
+            }
+        ),
+    )
+    message = queue.receive_messages(MessageAttributeNames=["All"])[0]
+    yield message
+
+
 @pytest.fixture
 def input_message_bitstream_dspace_post_error(mocked_sqs):
     queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue")

tests/test_submission.py

@@ -4,6 +4,7 @@
 import pytest
 from dspace import Bitstream, Item
 from freezegun import freeze_time
+from requests.exceptions import RequestException
 
 from submitter import errors
 from submitter.submission import Submission, prettify
@@ -181,3 +182,26 @@ def test_submit_bitstream_post_dspace_error(
         "Error occurred while posting bitstream 'test-file-01.pdf' to item in DSpace. "
         "Item '0000/item02' and any bitstreams already posted to it will be deleted"
     )
+
+
+def test_submit_dspace_unknown_api_error_logs_exception_and_raises_error(
+    caplog,
+    mocked_dspace,
+    test_client,
+    input_message_item_post_dspace_generic_500_error,
+):
+    submission = Submission.from_message(
+        input_message_item_post_dspace_generic_500_error
+    )
+    with pytest.raises(RequestException):
+        submission.submit(test_client)
+    # assert actual encountered exception is logged (for debugging purposes)
+    assert (
+        "Catastrophic error before or during request!  No response to parse."
+        in caplog.text
+    )
+    # assert unhandled exception encountered in submit flow logged
+    assert (
+        "Unexpected exception, aborting DSpace Submission Service processing"
+        in caplog.text
+    )