From 346a65c6951dc597a8b78ee8c077cdf0a7333127 Mon Sep 17 00:00:00 2001 From: abbas-khan10 <127417949+abbas-khan10@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:56:28 +0000 Subject: [PATCH] PRMDR:739 - Add dynamo fields for retry and state locking (#315) --- lambdas/enums/metadata_field_names.py | 4 +- lambdas/models/document_reference.py | 12 ++++-- lambdas/models/nhs_document_reference.py | 37 +++++++++++++------ lambdas/tests/unit/conftest.py | 2 +- .../unit/enums/test_metadata_field_names.py | 4 +- .../helpers/data/bulk_upload/test_data.py | 2 + .../unit/helpers/data/dynamo_responses.py | 15 ++++++-- .../unit/models/test_document_reference.py | 19 +++------- .../unit/services/test_bulk_upload_service.py | 5 +-- .../test_document_reference_search_service.py | 18 +++------ .../unit/services/test_document_service.py | 12 +----- .../test_lloyd_george_stitch_service.py | 4 ++ 12 files changed, 71 insertions(+), 63 deletions(-) diff --git a/lambdas/enums/metadata_field_names.py b/lambdas/enums/metadata_field_names.py index d0ef07ad9..e956da183 100755 --- a/lambdas/enums/metadata_field_names.py +++ b/lambdas/enums/metadata_field_names.py @@ -13,7 +13,9 @@ class DocumentReferenceMetadataFields(Enum): TYPE = "Type" VIRUS_SCANNER_RESULT = "VirusScannerResult" CURRENT_GP_ODS = "CurrentGpOds" - UPLOADED = 'Uploaded' + UPLOADED = "Uploaded" + UPLOADING = "Uploading" + LAST_UPDATED = "LastUpdated" @staticmethod def list() -> list[str]: diff --git a/lambdas/models/document_reference.py b/lambdas/models/document_reference.py index bee1fe8a3..c0635c8c8 100644 --- a/lambdas/models/document_reference.py +++ b/lambdas/models/document_reference.py @@ -42,8 +42,11 @@ class DocumentReference(BaseModel): alias=str(DocumentReferenceMetadataFields.CURRENT_GP_ODS.value), default=None ) - uploaded: bool = Field( - alias=str(DocumentReferenceMetadataFields.UPLOADED.value), default=False + uploaded: bool = Field(alias=str(DocumentReferenceMetadataFields.UPLOADED.value)) + uploading: bool = Field(alias=str(DocumentReferenceMetadataFields.UPLOADING.value)) + last_updated: str = Field( + alias=str(DocumentReferenceMetadataFields.LAST_UPDATED.value), + serialization_alias="lastUpdated", ) def get_file_name_path(self): @@ -85,7 +88,7 @@ def create_unique_filename(self, duplicates: int): return f"{self.get_base_name()}({duplicates}){self.get_file_extension()}" def __eq__(self, other): - if isinstance(self, DocumentReference): + if isinstance(other, DocumentReference): return ( self.id == other.id and self.content_type == other.content_type @@ -97,5 +100,8 @@ def __eq__(self, other): and self.ttl == other.ttl and self.virus_scanner_result == other.virus_scanner_result and self.current_gp_ods == other.current_gp_ods + and self.uploaded == other.uploaded + and self.uploading == other.uploading + and self.last_updated == other.last_updated ) return False diff --git a/lambdas/models/nhs_document_reference.py b/lambdas/models/nhs_document_reference.py index e5d44ff23..dce1da273 100644 --- a/lambdas/models/nhs_document_reference.py +++ b/lambdas/models/nhs_document_reference.py @@ -3,6 +3,8 @@ from enums.metadata_field_names import DocumentReferenceMetadataFields from pydantic import BaseModel +DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + class UploadRequestDocument(BaseModel): fileName: str @@ -20,20 +22,24 @@ def __init__( content_type: str = "application/pdf", current_gp_ods: str = "", ) -> None: + date_now = datetime.now(timezone.utc).strftime(DATE_FORMAT) + self.id = reference_id self.nhs_number = nhs_number self.content_type = content_type self.current_gp_ods = current_gp_ods self.file_name = file_name - self.created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.created = date_now self.s3_bucket_name = s3_bucket_name self.deleted = "" self.virus_scanner_result = "Not Scanned" self.file_location = f"s3://{self.s3_bucket_name}/{self.s3_file_key}" self.uploaded = False + self.uploading = False + self.last_updated = date_now def set_deleted(self) -> None: - self.deleted = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.deleted = datetime.now(timezone.utc).strftime(DATE_FORMAT) def set_virus_scanner_result(self, updated_virus_scanner_result) -> None: self.virus_scanner_result = updated_virus_scanner_result @@ -53,6 +59,8 @@ def to_dict(self): DocumentReferenceMetadataFields.VIRUS_SCANNER_RESULT.value: self.virus_scanner_result, DocumentReferenceMetadataFields.CURRENT_GP_ODS.value: self.current_gp_ods, DocumentReferenceMetadataFields.UPLOADED.value: self.uploaded, + DocumentReferenceMetadataFields.UPLOADING.value: self.uploading, + DocumentReferenceMetadataFields.LAST_UPDATED.value: self.last_updated, } return document_metadata @@ -61,13 +69,18 @@ def s3_file_key(self): return f"{self.nhs_number}/{self.id}" def __eq__(self, other): - return ( - self.id == other.id - and self.nhs_number == other.nhs_number - and self.content_type == other.content_type - and self.file_name == other.file_name - and self.created == other.created - and self.deleted == other.deleted - and self.virus_scanner_result == other.virus_scanner_result - and self.file_location == other.file_location - ) + if isinstance(other, NHSDocumentReference): + return ( + self.id == other.id + and self.nhs_number == other.nhs_number + and self.content_type == other.content_type + and self.file_name == other.file_name + and self.created == other.created + and self.deleted == other.deleted + and self.virus_scanner_result == other.virus_scanner_result + and self.file_location == other.file_location + and self.uploaded == other.uploaded + and self.uploading == other.uploading + and self.last_updated == other.last_updated + ) + return False diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index ef70da95b..a99e2782b 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -10,7 +10,7 @@ REGION_NAME = "eu-west-2" MOCK_TABLE_NAME = "test-table" -MOCK_BUCKET = "test_s3_bucket" +MOCK_BUCKET = "test-s3-bucket" MOCK_ARF_TABLE_NAME_ENV_NAME = "DOCUMENT_STORE_DYNAMODB_NAME" MOCK_ARF_BUCKET_ENV_NAME = "DOCUMENT_STORE_BUCKET_NAME" diff --git a/lambdas/tests/unit/enums/test_metadata_field_names.py b/lambdas/tests/unit/enums/test_metadata_field_names.py index 8be655e53..466cc1fa7 100755 --- a/lambdas/tests/unit/enums/test_metadata_field_names.py +++ b/lambdas/tests/unit/enums/test_metadata_field_names.py @@ -8,7 +8,7 @@ def test_can_get_one_field_name(): def test_returns_all_as_list(): subject = DocumentReferenceMetadataFields.list() - assert len(subject) == 12 + assert len(subject) == 14 assert DocumentReferenceMetadataFields.ID.value in subject assert DocumentReferenceMetadataFields.CONTENT_TYPE.value in subject assert DocumentReferenceMetadataFields.CREATED.value in subject @@ -21,3 +21,5 @@ def test_returns_all_as_list(): assert DocumentReferenceMetadataFields.VIRUS_SCANNER_RESULT.value in subject assert DocumentReferenceMetadataFields.CURRENT_GP_ODS.value in subject assert DocumentReferenceMetadataFields.UPLOADED.value in subject + assert DocumentReferenceMetadataFields.UPLOADING.value in subject + assert DocumentReferenceMetadataFields.LAST_UPDATED.value in subject diff --git a/lambdas/tests/unit/helpers/data/bulk_upload/test_data.py b/lambdas/tests/unit/helpers/data/bulk_upload/test_data.py index 6b2e4a11a..f2a52fcae 100644 --- a/lambdas/tests/unit/helpers/data/bulk_upload/test_data.py +++ b/lambdas/tests/unit/helpers/data/bulk_upload/test_data.py @@ -1,5 +1,6 @@ import os +from freezegun import freeze_time from models.nhs_document_reference import NHSDocumentReference from models.staging_metadata import MetadataFile, StagingMetadata from tests.unit.conftest import MOCK_LG_BUCKET, TEST_CURRENT_GP_ODS, TEST_OBJECT_KEY @@ -127,6 +128,7 @@ def build_test_sqs_message_from_nhs_number(nhs_number: str) -> dict: return build_test_sqs_message(staging_metadata) +@freeze_time("2024-01-01 12:00:00") def build_test_document_reference(file_name: str, nhs_number: str = "9000000009"): doc_ref = NHSDocumentReference( nhs_number=nhs_number, diff --git a/lambdas/tests/unit/helpers/data/dynamo_responses.py b/lambdas/tests/unit/helpers/data/dynamo_responses.py index 8410af8ad..8b5a3964d 100755 --- a/lambdas/tests/unit/helpers/data/dynamo_responses.py +++ b/lambdas/tests/unit/helpers/data/dynamo_responses.py @@ -5,33 +5,42 @@ "ContentType": "type", "Created": "2023-08-23T13:38:04.095Z", "Deleted": "", - "FileLocation": "s3://test_s3_bucket/test-key-123", + "FileLocation": "s3://test-s3-bucket/9000000009/test-key-123", "FileName": "document.csv", "NhsNumber": "9000000009", "VirusScannerResult": "Clean", "CurrentGpOds": "Y12345", + "Uploaded": "True", + "Uploading": "False", + "LastUpdated": "2023-08-23T13:38:04.095Z", }, { "ID": "4d8683b9-1665-40d2-8499-6e8302d507ff", "ContentType": "type", "Created": "2023-08-23T13:38:04.095Z", "Deleted": "", - "FileLocation": "s3://test_s3_bucket/test-key-223", + "FileLocation": "s3://test-s3-bucket/9000000009/test-key-223", "FileName": "results.pdf", "NhsNumber": "9000000009", "VirusScannerResult": "Clean", "CurrentGpOds": "Y12345", + "Uploaded": "True", + "Uploading": "False", + "LastUpdated": "2023-08-23T13:38:04.095Z", }, { "ID": "5d8683b9-1665-40d2-8499-6e8302d507ff", "ContentType": "type", "Created": "2023-08-24T14:38:04.095Z", "Deleted": "", - "FileLocation": "s3://test_s3_bucket/test-key-323", + "FileLocation": "s3://test-s3-bucket/9000000009/test-key-323", "FileName": "output.csv", "NhsNumber": "9000000009", "VirusScannerResult": "Clean", "CurrentGpOds": "Y12345", + "Uploaded": "True", + "Uploading": "False", + "LastUpdated": "2023-08-23T13:38:04.095Z", }, ], "Count": 3, diff --git a/lambdas/tests/unit/models/test_document_reference.py b/lambdas/tests/unit/models/test_document_reference.py index ae7892cac..a56e9a773 100644 --- a/lambdas/tests/unit/models/test_document_reference.py +++ b/lambdas/tests/unit/models/test_document_reference.py @@ -1,20 +1,11 @@ import pytest from models.document_reference import DocumentReference +from tests.unit.helpers.data.dynamo_responses import MOCK_SEARCH_RESPONSE from utils.exceptions import InvalidDocumentReferenceException -MOCK_DATA = { - "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff", - "ContentType": "type", - "Created": "2023-08-23T00:38:04.095Z", - "Deleted": "", - "FileLocation": "s3://test-bucket/9000000009/test-key-123", - "FileName": "document.csv", - "NhsNumber": "9000000009", - "VirusScannerResult": "Clean", - "CurrentGpOds": "Y12345", -} - -MOCK_DOCUMENT_REFERENCE = DocumentReference.model_validate(MOCK_DATA) +MOCK_DOCUMENT_REFERENCE = DocumentReference.model_validate( + MOCK_SEARCH_RESPONSE["Items"][0] +) def test_get_base_name(): @@ -34,7 +25,7 @@ def test_get_file_extension(): def test_get_file_bucket(): - expected = "test-bucket" + expected = "test-s3-bucket" actual = MOCK_DOCUMENT_REFERENCE.get_file_bucket() diff --git a/lambdas/tests/unit/services/test_bulk_upload_service.py b/lambdas/tests/unit/services/test_bulk_upload_service.py index 50ade6ec5..d4baef88b 100644 --- a/lambdas/tests/unit/services/test_bulk_upload_service.py +++ b/lambdas/tests/unit/services/test_bulk_upload_service.py @@ -625,6 +625,7 @@ def test_create_lg_records_and_copy_files(set_env, mocker, mock_uuid, repo_under ) +@freeze_time("2024-01-01 12:00:00") def test_convert_to_document_reference(set_env, mock_uuid, repo_under_test): TEST_STAGING_METADATA.retries = 0 repo_under_test.s3_repository.lg_bucket_name = "test_lg_s3_bucket" @@ -635,10 +636,6 @@ def test_convert_to_document_reference(set_env, mock_uuid, repo_under_test): current_gp_ods=TEST_CURRENT_GP_ODS, ) - # exclude the `created` timestamp from comparison - actual.created = "mock_timestamp" - expected.created = "mock_timestamp" - assert actual.__eq__(expected) diff --git a/lambdas/tests/unit/services/test_document_reference_search_service.py b/lambdas/tests/unit/services/test_document_reference_search_service.py index fdc88d08e..c1ed813a8 100644 --- a/lambdas/tests/unit/services/test_document_reference_search_service.py +++ b/lambdas/tests/unit/services/test_document_reference_search_service.py @@ -4,24 +4,16 @@ from botocore.exceptions import ClientError from models.document_reference import DocumentReference from services.document_reference_search_service import DocumentReferenceSearchService +from tests.unit.helpers.data.dynamo_responses import MOCK_SEARCH_RESPONSE from utils.exceptions import DynamoServiceException from utils.lambda_exceptions import DocumentRefSearchException -MOCK_DATA = { - "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff", - "ContentType": "type", - "Created": "2023-08-23T00:38:04.095Z", - "Deleted": "", - "FileLocation": "s3://test-bucket/9000000009/test-key-123", - "FileName": "document.csv", - "NhsNumber": "9000000009", - "VirusScannerResult": "Clean", -} - -MOCK_DOCUMENT_REFERENCE = [DocumentReference.model_validate(MOCK_DATA)] +MOCK_DOCUMENT_REFERENCE = [ + DocumentReference.model_validate(MOCK_SEARCH_RESPONSE["Items"][0]) +] EXPECTED_RESPONSE = { - "created": "2023-08-23T00:38:04.095Z", + "created": "2023-08-23T13:38:04.095Z", "fileName": "document.csv", "virusScannerResult": "Clean", } diff --git a/lambdas/tests/unit/services/test_document_service.py b/lambdas/tests/unit/services/test_document_service.py index 20d8fe2ef..b12285d13 100644 --- a/lambdas/tests/unit/services/test_document_service.py +++ b/lambdas/tests/unit/services/test_document_service.py @@ -19,17 +19,7 @@ MOCK_SEARCH_RESPONSE, ) -MOCK_DOCUMENT = { - "ID": "3d8683b9-1665-40d2-8499-6e8302d507ff", - "ContentType": "type", - "Created": "2023-08-23T13:38:04.095Z", - "Deleted": "", - "FileLocation": "s3://test_s3_bucket/test-key-123", - "FileName": "document.csv", - "NhsNumber": "9000000009", - "VirusScannerResult": "Clean", - "CurrentGpOds": "Y12345", -} +MOCK_DOCUMENT = MOCK_SEARCH_RESPONSE["Items"][0] @pytest.fixture diff --git a/lambdas/tests/unit/services/test_lloyd_george_stitch_service.py b/lambdas/tests/unit/services/test_lloyd_george_stitch_service.py index 8f6b7b426..0a1c6c8e0 100644 --- a/lambdas/tests/unit/services/test_lloyd_george_stitch_service.py +++ b/lambdas/tests/unit/services/test_lloyd_george_stitch_service.py @@ -34,6 +34,10 @@ def build_lg_doc_ref( "FileName": file_name, "NhsNumber": TEST_NHS_NUMBER, "VirusScannerResult": "Clean", + "CurrentGpOds": "Y12345", + "Uploaded": "True", + "Uploading": "False", + "LastUpdated": "2023-08-23T13:38:04.095Z", }, )