Skip to content

Commit

Permalink
Remove check letter from SDS identifier (#1280)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MebinAbraham
MebinAbraham authored Dec 14, 2023
1 parent b58802a commit 4eef48e
Show file tree
Hide file tree
Showing 28 changed files with 80 additions and 62 deletions.
3 changes: 3 additions & 0 deletions app/helpers/metadata_helpers.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
def get_ru_ref_without_check_letter(ru_ref: str) -> str:
"""Return the ru_ref without the check-letter"""
return ru_ref[:11]
9 changes: 8 additions & 1 deletion app/routes/session.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from app.data_models import QuestionnaireStore
from app.data_models.metadata_proxy import MetadataProxy
from app.globals import get_session_store, get_session_timeout_in_seconds
from app.helpers.metadata_helpers import get_ru_ref_without_check_letter
from app.helpers.template_helpers import (
DATA_LAYER_KEYS,
get_survey_config,
Expand Down Expand Up @@ -173,10 +174,16 @@ def _set_questionnaire_supplementary_data(
)
return

identifier = (
get_ru_ref_without_check_letter(metadata["ru_ref"])
if metadata["ru_ref"]
else metadata["qid"]
)

supplementary_data = get_supplementary_data_v1(
# Type ignore: survey_id and either ru_ref or qid are required for schemas that use supplementary data
dataset_id=new_sds_dataset_id,
identifier=metadata["ru_ref"] or metadata["qid"], # type: ignore
identifier=identifier, # type: ignore
survey_id=metadata["survey_id"], # type: ignore
)
logger.info(
Expand Down
2 changes: 1 addition & 1 deletion app/submitter/converter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def convert_answers(
'channel': 'RH',
'metadata': {
'user_id': '789473423',
'ru_ref': '432423423423'
'ru_ref': '12345678901A'
},
'data': [
...
Expand Down
3 changes: 2 additions & 1 deletion app/survey_config/business_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from flask_babel import lazy_gettext

from app.helpers.metadata_helpers import get_ru_ref_without_check_letter
from app.settings import read_file
from app.survey_config.link import HeaderLink, Link
from app.survey_config.survey_config import SurveyConfig
Expand Down Expand Up @@ -39,7 +40,7 @@ def _get_account_service_help_url(
# The ru_ref currently is concatenated with the check letter. Which upstream currently do not support.
# The first 11 characters represents the reporting unit reference.
# The 12th character is the check letter identifier.
"ru_ref": ru_ref[:11],
"ru_ref": get_ru_ref_without_check_letter(ru_ref),
}
return f"{self.base_url}/surveys/surveys-help?{urlencode(request_data)}"

Expand Down
2 changes: 1 addition & 1 deletion app/views/handlers/feedback.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ class FeedbackPayload:
"metadata": {
"ref_period_end_date": "2021-03-29",
"ref_period_start_date": "2021-03-01",
"ru_ref": "11110000022H",
"ru_ref": "12345678901A",
"user_id": "d98d78eb-d23a-494d-b67c-e770399de383"
},
"origin": "uk.gov.ons.edc.eq",
Expand Down
2 changes: 1 addition & 1 deletion tests/app/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def mocked_make_request_with_timeout(
def supplementary_data():
return {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
"note": {
"title": "Volume of total production",
"example": {
Expand Down
4 changes: 2 additions & 2 deletions tests/app/data_model/test_metadata_proxy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

METADATA_V1 = {
"schema_name": "1_0000",
"ru_ref": "432423423423",
"ru_ref": "12345678901A",
"response_id": "1",
"account_service_url": "account_service_url",
"tx_id": "tx_id",
Expand All @@ -26,7 +26,7 @@
"response_expires_at": "2023-04-24T10:46:32+00:00",
"survey_metadata": {
"data": {
"ru_ref": "432423423423",
"ru_ref": "12345678901A",
},
},
}
Expand Down
6 changes: 3 additions & 3 deletions tests/app/data_model/test_supplementary_data_store.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test_supplementary_data_serialisation(

def test_supplementary_data_deserialisation():
raw_data = {
"identifier": "12346789012A",
"identifier": "12345678901",
"items": {
"products": [
{"identifier": 89929001},
Expand All @@ -47,7 +47,7 @@ def test_supplementary_data_deserialisation():
assert deserialized.raw_data == make_immutable(raw_data)
assert deserialized.list_mappings == make_immutable(list_mappings)
assert deserialized._data_map == { # pylint: disable=protected-access
("identifier", None): "12346789012A",
("identifier", None): "12345678901",
("products", "item-1"): {"identifier": 89929001},
("products", "item-2"): {"identifier": "201630601"},
}
Expand All @@ -63,7 +63,7 @@ def test_empty_supplementary_data_deserialisation():
@pytest.mark.parametrize(
"identifier,list_item_id,selectors,expected",
[
("identifier", None, None, "12346789012A"),
("identifier", None, None, "12345678901"),
("note", None, ["title"], "Volume of total production"),
("products", "item-2", ["name"], "Other Minerals"),
("products", "item-1", ["value_sales", "answer_code"], "89929001"),
Expand Down
2 changes: 1 addition & 1 deletion tests/app/forms/test_field_factory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def test_invalid_field_type_raises_on_invalid():
"period_str": "2016-01-01",
"ref_p_start_date": "2016-02-02",
"ref_p_end_date": "2016-03-03",
"ru_ref": "432423423423",
"ru_ref": "12345678901A",
"ru_name": "Apple",
"return_by": "2016-07-07",
"case_id": "1234567890",
Expand Down
4 changes: 2 additions & 2 deletions tests/app/helpers/test_template_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,7 @@ def test_header_context(app: Flask, theme, survey_title, survey_config, expected
"itemsList": [
{
"title": "Help",
"url": f"{ACCOUNT_SERVICE_BASE_URL}/surveys/surveys-help?survey_ref=001&ru_ref=63782964754",
"url": f"{ACCOUNT_SERVICE_BASE_URL}/surveys/surveys-help?survey_ref=001&ru_ref=12345678901",
"id": "header-link-help",
},
{
Expand Down Expand Up @@ -480,7 +480,7 @@ def test_service_links_context(
if is_authenticated:
mocker.patch(
"app.helpers.template_helpers.get_metadata",
return_value=get_metadata({"ru_ref": "63782964754U", "tx_id": "tx_id"}),
return_value=get_metadata({"ru_ref": "12345678901A", "tx_id": "tx_id"}),
)

result = ContextHelper(
Expand Down
2 changes: 1 addition & 1 deletion tests/app/parser/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def fake_metadata_full_v2_business():
"ru_name": "Apple",
"return_by": "2016-07-07",
"case_ref": "1000000000000001",
"ru_ref": "123456789",
"ru_ref": "12345678901A",
"form_type": "I",
"response_expires_at": get_response_expires_at(),
}
Expand Down
30 changes: 15 additions & 15 deletions tests/app/parser/test_supplementary_data_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"survey_id": "123",
"data": {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
"items": {
"local_units": [
{
Expand Down Expand Up @@ -47,7 +47,7 @@ def test_invalid_supplementary_data_payload_raises_error():
validate_supplementary_data(
supplementary_data={},
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -58,7 +58,7 @@ def test_validate_supplementary_data_payload():
validated_payload = validate_supplementary_data_v1(
supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -70,7 +70,7 @@ def test_validate_supplementary_data_payload_incorrect_dataset_id():
validate_supplementary_data_v1(
supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
dataset_id="331507ca-1039-4624-a342-7cbc3630e217",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -85,7 +85,7 @@ def test_validate_supplementary_data_payload_incorrect_survey_id():
validate_supplementary_data_v1(
supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="234",
)

Expand Down Expand Up @@ -116,14 +116,14 @@ def test_supplementary_data_payload_with_no_items_is_validated():
"survey_id": "123",
"data": {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
},
}

validated_payload = validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -135,15 +135,15 @@ def test_validate_supplementary_data_payload_missing_survey_id():
"dataset_id": "44f1b432-9421-49e5-bd26-e63e18a30b69",
"data": {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
},
}

with pytest.raises(ValidationError) as error:
validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -157,14 +157,14 @@ def test_validate_supplementary_data_payload_with_unknown_field():
"some_field": "value",
"data": {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
},
}

validated_payload = validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -178,15 +178,15 @@ def test_validate_supplementary_data_invalid_schema_version():
"some_field": "value",
"data": {
"schema_version": "v2",
"identifier": "12346789012A",
"identifier": "12345678901",
},
}

with pytest.raises(ValidationError) as error:
validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="001",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -201,7 +201,7 @@ def test_validate_supplementary_data_payload_missing_identifier_in_items():
validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -217,7 +217,7 @@ def test_validate_supplementary_data_payload_invalid_identifier(invalid_identifi
validate_supplementary_data_v1(
supplementary_data=payload,
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand Down
4 changes: 2 additions & 2 deletions tests/app/services/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ def decrypted_mock_supplementary_data_payload():
"survey_id": "123",
"data": {
"schema_version": "v1",
"identifier": "12346789012A",
"identifier": "12345678901",
"items": {
"local_units": [
{
Expand Down Expand Up @@ -45,7 +45,7 @@ def encrypted_mock_supplementary_data_payload():
"dataset_id": "44f1b432-9421-49e5-bd26-e63e18a30b69",
"survey_id": "123",
# pylint: disable-next=line-too-long
"data": "eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkEyNTZHQ00iLCJraWQiOiJkZjg4ZmRhZDI2MTJhZTFlODA1NzExMjBlNmM2MzcxZjU1ODk2Njk2In0.HXHbiTFa2XCuaKfSX4HfONLAVfZxEZo6_072A-ilMwhQKVhmQzqhDOX9dRID1yeoJnYxcdVGRDPwD5QvVHY2qsIOGkO3Brh59GHvSVASSCR21od7DBvlv0LoCDGAVcPt5bwwMbmziGQNkyfNnZG1EwAFk09lWfXwapJDsKbu1uhbW-F8HOOFZT7vR3paUmeTS0ekITwod-eZTD-B7KwUnDhJSm12cl9ob8MHirCaTE3jB64iQe_GvgdSUs1n5HZnX1f7rDQEpWm0OeuxPbEDrFmU-9wBV6LAjszypxPhQ0pv76TMu-VhjNBgf5Xm0cMO8ycmubdBdyWZiVUcmG0or9Mw0QvDYA9th27RChZPs5Le0w9oTnp5p5qzQexpPdzcS9Niqwabwx-NyUvTYkkFWc9poGNtrane4Ei4AFlV1-nQ4A8wMSIuOOSteYVombw-FEo9FhIhnuU8qyHoy0EaW5D1PMcdsphSb2ybTsiEJfdwwpxWDiuMDUpW6c2It_uSEpLCFHj51pQ8_Ez44gDyRE41NpSVvOB6h-nOggICQKggBDimtAuOyunZ5jhQWlKAeX4vAwM4iUEJ82c9lLTb8EDmQDOj6os9PCmyrZku5FHRXXDDvyuQAEqB-ARRnWT47LZOmaeTn4RI92qcejOqTNSG80mCUMY1PPu1fqxOiG8.5ukQQc5IT8fHgMRJ.jDXFWkIV9PIIrAC6EnVu5joGNdlmM52xgEqXCocUvZflvT8xMNocGGDUD1S0_FbQlpSP8FJcby4-B88yzxLgwil7mBqGtE7kWjALPGgnV9DsjEs-OblFFUIY9dVaKMTwaXhSxpjEt4j9NUOo-uyKOQIg-ZT745_zbkFZhlScuHz0u1YYaZGN04Md-eDI38erkTrS3pe2aovjbrwc9s_FMbrHEKlnuAUwxGPUxLTJXpTavHdmhhWriaI6a9ymng6mcefKqpNfWXc-3MIY2k10JCoo2KlVyYRN0up6T4OW56mAoXf7dUNSC8PtcZ5J3rpJ4XUnLZNVSL6Kz9BAmXY5SpZlU9zIBxHsirG9fS8gV6bI_MeMwbMawMCylcArlINVGVwDnUIvNXIJplVgm1OTHL0TiZzEFvnqkro3mzvVL-ElFeYO0uegqN8cDGkcv_lpAHH33j-IPdG-8CYerTCSVrT9.vWWMrdJFFLTVBSiv3KhbiQ",
"data": "eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkEyNTZHQ00iLCJraWQiOiJkZjg4ZmRhZDI2MTJhZTFlODA1NzExMjBlNmM2MzcxZjU1ODk2Njk2In0.tK1Duhk7FmvMn7X6QaSvrpx5U0wOzOyzFwOaLUksCBrPI2J7Tnz1mRMoQla-ekuG-B0DLaQUWA74vgN44UhWbLd7fYbIwjt9TCgdfVyP7P9tOebe55xEHsbvMmRf7F1F37KjPXXk7usDkFuuzl_fjhLwmrjNn90YA93QLy1PbGiMcC8JFYDwLL-vaWslB-VGgKEdud4LCY80xvwxzxxHYtpEQ2NpBXt9zpodwQCFqn8LHHGN80h8_-TE2gOInWccRd7GsyzvH5hGb-wxmgodJQFrPse4v6VBgTJbixMFvxulZqAyYULzU7LiFGSQZqrhqqL4-UXWRmeJX5aYR9AtzO2kGl6gVDZ2d0jGLuHwOSiJ9CLtvltKU8ai0ZfN-qUY6ZnV2lZ7wq_fZrat2-VdxE-ktMy6owGBD3DwHcd_QwAPlDFDEJeAS1G4JKuyfCb_siM5wLd1_HV_6kG2IcyAtt0usRj2D36N2K2lufJjJJlDh2QJm6r5H1pGci8AR7YCJ7nAbmlyMJ7RLJ65TswcvWT7A8jQnVBZT2NCL3_WZ9o2cGj9HixzATzVPkP54LWPTgDI-Q0-nLaBfkHFcR9-6v9f_DFW_zcj2yFfu93HRSc4GSjGs7x0dZkpY-JLFqcNzxcEeUMODUgeE9QZAIF2VsS7NcodbpqJJlOBbevJeZU.Z6Gte9FcRQfo3vos.kwL0nw0aZj6hYugPWt5oa4j-SwdBhDl-ynd6UDtqf1WFDLZcS_DBaXXT2tB7vh_zHNJaMhkJfZV4iT1jr6Wat8pDhL4c_IrclxMLagU22dST6aZcCBkVtvVUCB5dt0AO64NUBysiDf8nTnJyzLnC-pMXvc1QXSS5OMdrFdLPm3jaTAT7y1NYNtIQyRrLZj7PayRQK5w5JS3clq3BSn780rECVxM4FLuDsO0hft-feDKW700Xe4HAeXqaLwADM3bUXMeriiX2CvdTkBB2gv2uBQdR6FIVwGPDbTvXHf1bvTCLmaIT-Ki3pDh_BIrZhgg9ZO8Pft5ZEWWeQR0kF-2eglkro90AWSzUFrQO5YsYsQ-YdjoajeyyiQTg1zW3LyGM3F7-EDKrAZzUgiUhK9UC1Lfheg7Owg7aG-gYN3N6UU5ngnXegcJtNEnoULOLnck0aLS7Ku0v19tw4ObGPW28KZo.Ewzo-AyJbBMCWPqlhjhhAw",
}


Expand Down
16 changes: 8 additions & 8 deletions tests/app/services/test_request_supplementary_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def test_get_supplementary_data_v1_200(
)
loaded_supplementary_data = get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand Down Expand Up @@ -66,7 +66,7 @@ def test_get_supplementary_data_v1_non_200(
with pytest.raises(SupplementaryDataRequestFailed) as exc:
get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -82,7 +82,7 @@ def test_get_supplementary_data_v1_request_failed(app: Flask):
with pytest.raises(SupplementaryDataRequestFailed) as exc:
get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -105,7 +105,7 @@ def test_get_supplementary_data_v1_retries_timeout_error(
try:
supplementary_data = get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)
except SupplementaryDataRequestFailed:
Expand Down Expand Up @@ -137,7 +137,7 @@ def test_get_supplementary_data_v1_retries_transient_error(
try:
supplementary_data = get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)
except SupplementaryDataRequestFailed:
Expand All @@ -163,7 +163,7 @@ def test_get_supplementary_data_v1_max_retries(app: Flask, mocker):
with pytest.raises(SupplementaryDataRequestFailed) as exc:
get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand Down Expand Up @@ -219,7 +219,7 @@ def test_get_supplementary_data_v1_raises_missing_supplementary_data_key_error_w
with pytest.raises(MissingSupplementaryDataKey):
get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)

Expand All @@ -246,7 +246,7 @@ def test_get_supplementary_data_v1_with_gcp_authentication(

get_supplementary_data_v1(
dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
identifier="12346789012A",
identifier="12345678901",
survey_id="123",
)
mock_oidc_service.get_credentials.assert_called_once_with(
Expand Down
4 changes: 2 additions & 2 deletions tests/app/submitter/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"period_str": "2016-01-01",
"ref_p_start_date": "2016-02-02",
"ref_p_end_date": "2016-03-03",
"ru_ref": "432423423423",
"ru_ref": "12345678901A",
"response_id": "1234567890123456",
"ru_name": "Apple",
"return_by": "2016-07-07",
Expand Down Expand Up @@ -56,7 +56,7 @@
"period_str": "2016-01-01",
"ref_p_start_date": "2016-02-02",
"ref_p_end_date": "2016-03-03",
"ru_ref": "432423423423",
"ru_ref": "12345678901A",
"ru_name": "Apple",
"case_type": "SPG",
"form_type": "I",
Expand Down
Loading

0 comments on commit 4eef48e

Please sign in to comment.