PRMP-305 - "CurrentGpOds" field is empty when LG file uploaded from f…

…ront end (#382)

* Take pds call logic out of validate_lg_files, add a method to append ods code to doc ref

* minor fix

* [PRMP-305] Add currentGpOds for LG upload files in createDocRef service

* move a shared test data to conftest

* (WIP) [PRMP-305] check for LG doc existence and retrieve patient detail earlier in createDocRef lambda

* [PRMP-305] Add currentGpOds during doc ref creation, remove method that patch ods code to LG dictionary

* [PRMP-305] remove duplicated calls to pds

* bump up urllib3 version (to address vulnerabilities)

* fix dep conflict

lambdas/requirements/layers/requirements_core_lambda_layer.txt

@@ -1,7 +1,7 @@
 PyJWT==2.8.0
 PyYAML==6.0.1
-boto3==1.33.11
-botocore==1.33.11
+boto3==1.34.128
+botocore==1.34.128
 certifi==2023.7.22
 charset-normalizer==3.2.0
 cryptography==42.0.4
@@ -17,8 +17,8 @@ pypdf==3.17.2
 python-dateutil==2.8.2
 requests==2.32.0
 responses==0.23.1
-s3transfer==0.8.2
+s3transfer==0.10.1
 setuptools==65.5.1
 six==1.16.0
 types-PyYAML==6.0.12.11
-urllib3==2.0.7
+urllib3==2.2.2

lambdas/requirements/requirements_github_runner.txt

@@ -1 +1 @@
-boto3==1.33.11
+boto3==1.34.128

lambdas/services/create_document_reference_service.py

@@ -14,7 +14,11 @@
 from utils.common_query_filters import NotDeleted
 from utils.exceptions import InvalidResourceIdException
 from utils.lambda_exceptions import CreateDocumentRefException
-from utils.lloyd_george_validator import LGInvalidFilesException, validate_lg_files
+from utils.lloyd_george_validator import (
+    LGInvalidFilesException,
+    getting_patient_info_from_pds,
+    validate_lg_files,
+)
 from utils.utilities import create_reference_id, validate_nhs_number
 
 FAILED_CREATE_REFERENCE_MESSAGE = "Create document reference failed"
@@ -51,10 +55,24 @@ def create_document_reference_request(
         lg_documents_dict_format: list = []
         url_responses = {}
 
+        upload_request_documents = self.parse_documents_list(documents_list)
+
+        has_lg_document = any(
+            document.docType == SupportedDocumentTypes.LG
+            for document in upload_request_documents
+        )
+
+        current_gp_ods = ""
+        if has_lg_document:
+            pds_patient_details = getting_patient_info_from_pds(nhs_number)
+            current_gp_ods = pds_patient_details.general_practice_ods
+
         try:
             validate_nhs_number(nhs_number)
-            for document in documents_list:
-                document_reference = self.prepare_doc_object(nhs_number, document)
+            for validated_doc in upload_request_documents:
+                document_reference = self.prepare_doc_object(
+                    nhs_number, current_gp_ods, validated_doc
+                )
 
                 match document_reference.doc_type:
                     case SupportedDocumentTypes.ARF.value:
@@ -77,7 +95,7 @@ def create_document_reference_request(
                 )
 
             if lg_documents:
-                validate_lg_files(lg_documents, nhs_number)
+                validate_lg_files(lg_documents, pds_patient_details)
                 self.check_existing_lloyd_george_records(nhs_number)
 
                 self.create_reference_in_dynamodb(
@@ -98,32 +116,43 @@ def create_document_reference_request(
             )
             raise CreateDocumentRefException(400, LambdaError.CreateDocFiles)
 
+    def parse_documents_list(
+        self, document_list: list[dict]
+    ) -> list[UploadRequestDocument]:
+        upload_request_document_list = []
+        for document in document_list:
+            try:
+                validated_doc: UploadRequestDocument = (
+                    UploadRequestDocument.model_validate(document)
+                )
+                upload_request_document_list.append(validated_doc)
+            except ValidationError as e:
+                logger.error(
+                    f"{LambdaError.CreateDocNoParse.to_str()} :{str(e)}",
+                    {"Result": FAILED_CREATE_REFERENCE_MESSAGE},
+                )
+                raise CreateDocumentRefException(400, LambdaError.CreateDocNoParse)
+
+        return upload_request_document_list
+
     def prepare_doc_object(
-        self, nhs_number: str, document: dict
+        self, nhs_number: str, current_gp_ods: str, validated_doc: UploadRequestDocument
     ) -> NHSDocumentReference:
-        try:
-            validated_doc: UploadRequestDocument = UploadRequestDocument.model_validate(
-                document
-            )
-        except ValidationError as e:
-            logger.error(
-                f"{LambdaError.CreateDocNoParse.to_str()} :{str(e)}",
-                {"Result": FAILED_CREATE_REFERENCE_MESSAGE},
-            )
-            raise CreateDocumentRefException(400, LambdaError.CreateDocNoParse)
 
         logger.info(PROVIDED_DOCUMENT_SUPPORTED_MESSAGE)
 
         if validated_doc.docType == SupportedDocumentTypes.LG.value:
             document_reference = self.create_document_reference(
                 nhs_number,
+                current_gp_ods,
                 validated_doc,
                 s3_bucket_name=self.staging_bucket_name,
                 sub_folder=self.upload_sub_folder,
             )
         elif validated_doc.docType == SupportedDocumentTypes.ARF.value:
             document_reference = self.create_document_reference(
                 nhs_number,
+                current_gp_ods,
                 validated_doc,
                 s3_bucket_name=self.arf_bucket_name,
             )
@@ -139,14 +168,16 @@ def prepare_doc_object(
     def create_document_reference(
         self,
         nhs_number: str,
+        current_gp_ods: str,
         validated_doc: UploadRequestDocument,
-        s3_bucket_name,
+        s3_bucket_name: str,
         sub_folder="",
     ) -> NHSDocumentReference:
         s3_object_key = create_reference_id()
 
         document_reference = NHSDocumentReference(
             nhs_number=nhs_number,
+            current_gp_ods=current_gp_ods,
             s3_bucket_name=s3_bucket_name,
             sub_folder=sub_folder,
             reference_id=s3_object_key,

lambdas/tests/unit/conftest.py

@@ -6,6 +6,7 @@
 
 import pytest
 from models.document_reference import DocumentReference
+from models.pds_models import PatientDetails
 from pydantic import ValidationError
 
 REGION_NAME = "eu-west-2"
@@ -158,6 +159,24 @@ def set_env(monkeypatch):
     )
 
 
+EXPECTED_PARSED_PATIENT_BASE_CASE = PatientDetails(
+    givenName=["Jane"],
+    familyName="Smith",
+    birthDate="2010-10-22",
+    postalCode="LS1 6AE",
+    nhsNumber="9000000009",
+    superseded=False,
+    restricted=False,
+    generalPracticeOds="Y12345",
+    active=True,
+)
+
+
+@pytest.fixture
+def mock_patient_details():
+    yield EXPECTED_PARSED_PATIENT_BASE_CASE
+
+
 @pytest.fixture(scope="session", autouse=True)
 def logger_mocker():
     with mock.patch("utils.audit_logging_setup.SensitiveAuditService.emit") as _fixture:

lambdas/tests/unit/helpers/data/create_document_reference.py

@@ -1,3 +1,4 @@
+from models.nhs_document_reference import UploadRequestDocument
 from tests.unit.conftest import TEST_NHS_NUMBER
 from tests.unit.helpers.data.s3_responses import MOCK_PRESIGNED_URL_RESPONSE
 
@@ -60,6 +61,15 @@
     },
 ]
 
+PARSED_LG_FILE_LIST = [
+    UploadRequestDocument(
+        fileName=f"{i}of3_Lloyd_George_Record_[Joe Bloggs]_[{TEST_NHS_NUMBER}]_[25-12-2019].pdf",
+        contentType="application/pdf",
+        docType="LG",
+    )
+    for i in [1, 2, 3]
+]
+
 LG_MOCK_EVENT_BODY = {
     "resourceType": "DocumentReference",
     "subject": {"identifier": {"value": TEST_NHS_NUMBER}},
@@ -161,6 +171,15 @@
     },
 ]
 
+PARSED_ARF_FILE_LIST = [
+    UploadRequestDocument(
+        fileName=f"test{i}.txt",
+        contentType="text/plain",
+        docType="ARF",
+    )
+    for i in [1, 2, 3]
+]
+
 ARF_MOCK_EVENT_BODY = {
     "resourceType": "DocumentReference",
     "subject": {"identifier": {"value": TEST_NHS_NUMBER}},

lambdas/tests/unit/helpers/data/test_documents.py

@@ -1,7 +1,14 @@
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
+from enums.supported_document_types import SupportedDocumentTypes
 from models.document_reference import DocumentReference
-from tests.unit.conftest import MOCK_LG_BUCKET
+from models.nhs_document_reference import NHSDocumentReference
+from tests.unit.conftest import (
+    MOCK_LG_BUCKET,
+    MOCK_LG_STAGING_STORE_BUCKET_ENV_NAME,
+    TEST_NHS_NUMBER,
+    TEST_UUID,
+)
 from tests.unit.helpers.data.dynamo_responses import MOCK_SEARCH_RESPONSE
 
 
@@ -12,7 +19,9 @@ def create_test_doc_store_refs():
     ]
 
 
-def create_test_lloyd_george_doc_store_refs(override: Optional[Dict] = None):
+def create_test_lloyd_george_doc_store_refs(
+    override: Optional[Dict] = None,
+) -> List[DocumentReference]:
     refs = create_test_doc_store_refs()
 
     filename_1 = "1of3_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[30-12-2019].pdf"
@@ -29,3 +38,39 @@ def create_test_lloyd_george_doc_store_refs(override: Optional[Dict] = None):
     if override:
         refs = [doc_ref.model_copy(update=override) for doc_ref in refs]
     return refs
+
+
+def create_test_doc_refs(
+    override: Optional[Dict] = None, file_names: Optional[List[str]] = None
+) -> List[NHSDocumentReference]:
+    if not file_names:
+        file_names = [
+            f"{i}of3_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[30-12-2019].pdf"
+            for i in range(1, 4)
+        ]
+
+    override = override or {}
+    arguments = {
+        "nhs_number": TEST_NHS_NUMBER,
+        "s3_bucket_name": MOCK_LG_STAGING_STORE_BUCKET_ENV_NAME,
+        "sub_folder": "upload",
+        "reference_id": TEST_UUID,
+        "content_type": "application/pdf",
+        "doc_type": SupportedDocumentTypes.LG.value,
+        "uploading": True,
+        **override,
+    }
+
+    list_of_doc_ref = [
+        NHSDocumentReference(file_name=file_name, **arguments)
+        for file_name in file_names
+    ]
+
+    return list_of_doc_ref
+
+
+def create_test_doc_refs_as_dict(
+    override: Optional[Dict] = None, file_names: Optional[List[str]] = None
+) -> List[Dict]:
+    test_doc_refs = create_test_doc_refs(override, file_names)
+    return [doc_ref.to_dict() for doc_ref in test_doc_refs]

lambdas/tests/unit/models/test_pds_models.py

@@ -1,5 +1,6 @@
 from freezegun import freeze_time
 from models.pds_models import PatientDetails
+from tests.unit.conftest import EXPECTED_PARSED_PATIENT_BASE_CASE
 from tests.unit.helpers.data.pds.pds_patient_response import (
     PDS_PATIENT,
     PDS_PATIENT_NO_GIVEN_NAME_IN_CURRENT_NAME,
@@ -14,18 +15,6 @@
 from tests.unit.helpers.data.pds.utils import create_patient
 from utils.utilities import validate_nhs_number
 
-EXPECTED_PARSED_PATIENT_BASE_CASE = PatientDetails(
-    givenName=["Jane"],
-    familyName="Smith",
-    birthDate="2010-10-22",
-    postalCode="LS1 6AE",
-    nhsNumber="9000000009",
-    superseded=False,
-    restricted=False,
-    generalPracticeOds="Y12345",
-    active=True,
-)
-
 
 def test_validate_nhs_number_with_valid_id_returns_true():
     nhs_number = "0000000000"