PRMDR:739 - Add dynamo fields for retry and state locking (#315)

lambdas/enums/metadata_field_names.py

@@ -13,7 +13,9 @@ class DocumentReferenceMetadataFields(Enum):
     TYPE = "Type"
     VIRUS_SCANNER_RESULT = "VirusScannerResult"
     CURRENT_GP_ODS = "CurrentGpOds"
-    UPLOADED = 'Uploaded'
+    UPLOADED = "Uploaded"
+    UPLOADING = "Uploading"
+    LAST_UPDATED = "LastUpdated"
 
     @staticmethod
     def list() -> list[str]:

lambdas/models/document_reference.py

@@ -42,8 +42,11 @@ class DocumentReference(BaseModel):
         alias=str(DocumentReferenceMetadataFields.CURRENT_GP_ODS.value), default=None
     )
 
-    uploaded: bool = Field(
-        alias=str(DocumentReferenceMetadataFields.UPLOADED.value), default=False
+    uploaded: bool = Field(alias=str(DocumentReferenceMetadataFields.UPLOADED.value))
+    uploading: bool = Field(alias=str(DocumentReferenceMetadataFields.UPLOADING.value))
+    last_updated: str = Field(
+        alias=str(DocumentReferenceMetadataFields.LAST_UPDATED.value),
+        serialization_alias="lastUpdated",
     )
 
     def get_file_name_path(self):
@@ -85,7 +88,7 @@ def create_unique_filename(self, duplicates: int):
         return f"{self.get_base_name()}({duplicates}){self.get_file_extension()}"
 
     def __eq__(self, other):
-        if isinstance(self, DocumentReference):
+        if isinstance(other, DocumentReference):
             return (
                 self.id == other.id
                 and self.content_type == other.content_type
@@ -97,5 +100,8 @@ def __eq__(self, other):
                 and self.ttl == other.ttl
                 and self.virus_scanner_result == other.virus_scanner_result
                 and self.current_gp_ods == other.current_gp_ods
+                and self.uploaded == other.uploaded
+                and self.uploading == other.uploading
+                and self.last_updated == other.last_updated
             )
         return False

lambdas/models/nhs_document_reference.py

@@ -3,6 +3,8 @@
 from enums.metadata_field_names import DocumentReferenceMetadataFields
 from pydantic import BaseModel
 
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
 
 class UploadRequestDocument(BaseModel):
     fileName: str
@@ -20,20 +22,24 @@ def __init__(
         content_type: str = "application/pdf",
         current_gp_ods: str = "",
     ) -> None:
+        date_now = datetime.now(timezone.utc).strftime(DATE_FORMAT)
+
         self.id = reference_id
         self.nhs_number = nhs_number
         self.content_type = content_type
         self.current_gp_ods = current_gp_ods
         self.file_name = file_name
-        self.created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        self.created = date_now
         self.s3_bucket_name = s3_bucket_name
         self.deleted = ""
         self.virus_scanner_result = "Not Scanned"
         self.file_location = f"s3://{self.s3_bucket_name}/{self.s3_file_key}"
         self.uploaded = False
+        self.uploading = False
+        self.last_updated = date_now
 
     def set_deleted(self) -> None:
-        self.deleted = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        self.deleted = datetime.now(timezone.utc).strftime(DATE_FORMAT)
 
     def set_virus_scanner_result(self, updated_virus_scanner_result) -> None:
         self.virus_scanner_result = updated_virus_scanner_result
@@ -53,6 +59,8 @@ def to_dict(self):
             DocumentReferenceMetadataFields.VIRUS_SCANNER_RESULT.value: self.virus_scanner_result,
             DocumentReferenceMetadataFields.CURRENT_GP_ODS.value: self.current_gp_ods,
             DocumentReferenceMetadataFields.UPLOADED.value: self.uploaded,
+            DocumentReferenceMetadataFields.UPLOADING.value: self.uploading,
+            DocumentReferenceMetadataFields.LAST_UPDATED.value: self.last_updated,
         }
         return document_metadata
 
@@ -61,13 +69,18 @@ def s3_file_key(self):
         return f"{self.nhs_number}/{self.id}"
 
     def __eq__(self, other):
-        return (
-            self.id == other.id
-            and self.nhs_number == other.nhs_number
-            and self.content_type == other.content_type
-            and self.file_name == other.file_name
-            and self.created == other.created
-            and self.deleted == other.deleted
-            and self.virus_scanner_result == other.virus_scanner_result
-            and self.file_location == other.file_location
-        )
+        if isinstance(other, NHSDocumentReference):
+            return (
+                self.id == other.id
+                and self.nhs_number == other.nhs_number
+                and self.content_type == other.content_type
+                and self.file_name == other.file_name
+                and self.created == other.created
+                and self.deleted == other.deleted
+                and self.virus_scanner_result == other.virus_scanner_result
+                and self.file_location == other.file_location
+                and self.uploaded == other.uploaded
+                and self.uploading == other.uploading
+                and self.last_updated == other.last_updated
+            )
+        return False

lambdas/tests/unit/conftest.py

@@ -10,7 +10,7 @@
 REGION_NAME = "eu-west-2"
 
 MOCK_TABLE_NAME = "test-table"
-MOCK_BUCKET = "test_s3_bucket"
+MOCK_BUCKET = "test-s3-bucket"
 
 MOCK_ARF_TABLE_NAME_ENV_NAME = "DOCUMENT_STORE_DYNAMODB_NAME"
 MOCK_ARF_BUCKET_ENV_NAME = "DOCUMENT_STORE_BUCKET_NAME"

lambdas/tests/unit/enums/test_metadata_field_names.py

@@ -8,7 +8,7 @@ def test_can_get_one_field_name():
 
 def test_returns_all_as_list():
     subject = DocumentReferenceMetadataFields.list()
-    assert len(subject) == 12
+    assert len(subject) == 14
     assert DocumentReferenceMetadataFields.ID.value in subject
     assert DocumentReferenceMetadataFields.CONTENT_TYPE.value in subject
     assert DocumentReferenceMetadataFields.CREATED.value in subject
@@ -21,3 +21,5 @@ def test_returns_all_as_list():
     assert DocumentReferenceMetadataFields.VIRUS_SCANNER_RESULT.value in subject
     assert DocumentReferenceMetadataFields.CURRENT_GP_ODS.value in subject
     assert DocumentReferenceMetadataFields.UPLOADED.value in subject
+    assert DocumentReferenceMetadataFields.UPLOADING.value in subject
+    assert DocumentReferenceMetadataFields.LAST_UPDATED.value in subject

lambdas/tests/unit/helpers/data/bulk_upload/test_data.py

@@ -1,5 +1,6 @@
 import os
 
+from freezegun import freeze_time
 from models.nhs_document_reference import NHSDocumentReference
 from models.staging_metadata import MetadataFile, StagingMetadata
 from tests.unit.conftest import MOCK_LG_BUCKET, TEST_CURRENT_GP_ODS, TEST_OBJECT_KEY
@@ -127,6 +128,7 @@ def build_test_sqs_message_from_nhs_number(nhs_number: str) -> dict:
     return build_test_sqs_message(staging_metadata)
 
 
+@freeze_time("2024-01-01 12:00:00")
 def build_test_document_reference(file_name: str, nhs_number: str = "9000000009"):
     doc_ref = NHSDocumentReference(
         nhs_number=nhs_number,

lambdas/tests/unit/helpers/data/dynamo_responses.py

@@ -5,33 +5,42 @@
             "ContentType": "type",
             "Created": "2023-08-23T13:38:04.095Z",
             "Deleted": "",
-            "FileLocation": "s3://test_s3_bucket/test-key-123",
+            "FileLocation": "s3://test-s3-bucket/9000000009/test-key-123",
             "FileName": "document.csv",
             "NhsNumber": "9000000009",
             "VirusScannerResult": "Clean",
             "CurrentGpOds": "Y12345",
+            "Uploaded": "True",
+            "Uploading": "False",
+            "LastUpdated": "2023-08-23T13:38:04.095Z",
         },
         {
             "ID": "4d8683b9-1665-40d2-8499-6e8302d507ff",
             "ContentType": "type",
             "Created": "2023-08-23T13:38:04.095Z",
             "Deleted": "",
-            "FileLocation": "s3://test_s3_bucket/test-key-223",
+            "FileLocation": "s3://test-s3-bucket/9000000009/test-key-223",
             "FileName": "results.pdf",
             "NhsNumber": "9000000009",
             "VirusScannerResult": "Clean",
             "CurrentGpOds": "Y12345",
+            "Uploaded": "True",
+            "Uploading": "False",
+            "LastUpdated": "2023-08-23T13:38:04.095Z",
         },
         {
             "ID": "5d8683b9-1665-40d2-8499-6e8302d507ff",
             "ContentType": "type",
             "Created": "2023-08-24T14:38:04.095Z",
             "Deleted": "",
-            "FileLocation": "s3://test_s3_bucket/test-key-323",
+            "FileLocation": "s3://test-s3-bucket/9000000009/test-key-323",
             "FileName": "output.csv",
             "NhsNumber": "9000000009",
             "VirusScannerResult": "Clean",
             "CurrentGpOds": "Y12345",
+            "Uploaded": "True",
+            "Uploading": "False",
+            "LastUpdated": "2023-08-23T13:38:04.095Z",
         },
     ],
     "Count": 3,

lambdas/tests/unit/models/test_document_reference.py

@@ -1,20 +1,11 @@
 import pytest
 from models.document_reference import DocumentReference
+from tests.unit.helpers.data.dynamo_responses import MOCK_SEARCH_RESPONSE
 from utils.exceptions import InvalidDocumentReferenceException
 
-MOCK_DATA = {
-    "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff",
-    "ContentType": "type",
-    "Created": "2023-08-23T00:38:04.095Z",
-    "Deleted": "",
-    "FileLocation": "s3://test-bucket/9000000009/test-key-123",
-    "FileName": "document.csv",
-    "NhsNumber": "9000000009",
-    "VirusScannerResult": "Clean",
-    "CurrentGpOds": "Y12345",
-}
-
-MOCK_DOCUMENT_REFERENCE = DocumentReference.model_validate(MOCK_DATA)
+MOCK_DOCUMENT_REFERENCE = DocumentReference.model_validate(
+    MOCK_SEARCH_RESPONSE["Items"][0]
+)
 
 
 def test_get_base_name():
@@ -34,7 +25,7 @@ def test_get_file_extension():
 
 
 def test_get_file_bucket():
-    expected = "test-bucket"
+    expected = "test-s3-bucket"
 
     actual = MOCK_DOCUMENT_REFERENCE.get_file_bucket()
 

lambdas/tests/unit/services/test_bulk_upload_service.py

@@ -625,6 +625,7 @@ def test_create_lg_records_and_copy_files(set_env, mocker, mock_uuid, repo_under
     )
 
 
+@freeze_time("2024-01-01 12:00:00")
 def test_convert_to_document_reference(set_env, mock_uuid, repo_under_test):
     TEST_STAGING_METADATA.retries = 0
     repo_under_test.s3_repository.lg_bucket_name = "test_lg_s3_bucket"
@@ -635,10 +636,6 @@ def test_convert_to_document_reference(set_env, mock_uuid, repo_under_test):
         current_gp_ods=TEST_CURRENT_GP_ODS,
     )
 
-    # exclude the `created` timestamp from comparison
-    actual.created = "mock_timestamp"
-    expected.created = "mock_timestamp"
-
     assert actual.__eq__(expected)
 
 

lambdas/tests/unit/services/test_document_reference_search_service.py

@@ -4,24 +4,16 @@
 from botocore.exceptions import ClientError
 from models.document_reference import DocumentReference
 from services.document_reference_search_service import DocumentReferenceSearchService
+from tests.unit.helpers.data.dynamo_responses import MOCK_SEARCH_RESPONSE
 from utils.exceptions import DynamoServiceException
 from utils.lambda_exceptions import DocumentRefSearchException
 
-MOCK_DATA = {
-    "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff",
-    "ContentType": "type",
-    "Created": "2023-08-23T00:38:04.095Z",
-    "Deleted": "",
-    "FileLocation": "s3://test-bucket/9000000009/test-key-123",
-    "FileName": "document.csv",
-    "NhsNumber": "9000000009",
-    "VirusScannerResult": "Clean",
-}
-
-MOCK_DOCUMENT_REFERENCE = [DocumentReference.model_validate(MOCK_DATA)]
+MOCK_DOCUMENT_REFERENCE = [
+    DocumentReference.model_validate(MOCK_SEARCH_RESPONSE["Items"][0])
+]
 
 EXPECTED_RESPONSE = {
-    "created": "2023-08-23T00:38:04.095Z",
+    "created": "2023-08-23T13:38:04.095Z",
     "fileName": "document.csv",
     "virusScannerResult": "Clean",
 }

lambdas/tests/unit/services/test_document_service.py

@@ -19,17 +19,7 @@
     MOCK_SEARCH_RESPONSE,
 )
 
-MOCK_DOCUMENT = {
-    "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff",
-    "ContentType": "type",
-    "Created": "2023-08-23T13:38:04.095Z",
-    "Deleted": "",
-    "FileLocation": "s3://test_s3_bucket/test-key-123",
-    "FileName": "document.csv",
-    "NhsNumber": "9000000009",
-    "VirusScannerResult": "Clean",
-    "CurrentGpOds": "Y12345",
-}
+MOCK_DOCUMENT = MOCK_SEARCH_RESPONSE["Items"][0]
 
 
 @pytest.fixture

lambdas/tests/unit/services/test_lloyd_george_stitch_service.py

@@ -34,6 +34,10 @@ def build_lg_doc_ref(
             "FileName": file_name,
             "NhsNumber": TEST_NHS_NUMBER,
             "VirusScannerResult": "Clean",
+            "CurrentGpOds": "Y12345",
+            "Uploaded": "True",
+            "Uploading": "False",
+            "LastUpdated": "2023-08-23T13:38:04.095Z",
         },
     )