From f610e5f6dc180b1394d17d6bb63e1159d40fbae6 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:06:14 +0200 Subject: [PATCH 01/32] Update waypoint.hcl --- waypoint.hcl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/waypoint.hcl b/waypoint.hcl index e94c78e..589aefa 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -1,16 +1,17 @@ -project = "prosanteconnect/pscextract" +project = "prosanteconnect/${workspace.name}/pscextract" # Labels can be specified for organizational purposes. labels = { "domaine" = "psc" } runner { enabled = true + profile = "secpsc-${workspace.name}" data_source "git" { url = "https://github.com/prosanteconnect/pscextract.git" - ref = var.datacenter + ref = "${workspace.name}" } poll { - enabled = true + enabled = false } } @@ -28,9 +29,10 @@ app "prosanteconnect/pscextract" { # Uncomment below to use a remote docker registry to push your built images. registry { use "docker" { - image = "${var.registry_path}/pscextract" + image = "${var.registry_username}/pscextract" tag = gitrefpretty() - encoded_auth = filebase64("/secrets/dockerAuth.json") + username = var.registry_username + password = var.registry_password } } } @@ -40,6 +42,7 @@ app "prosanteconnect/pscextract" { use "nomad-jobspec" { jobspec = templatefile("${path.app}/pscextract.nomad.tpl", { datacenter = var.datacenter + nomad_namespace = var.nomad_namespace registry_path = var.registry_path }) } From fc2fc50fe3c55db12356a3718368253584656f63 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:13:02 +0200 Subject: [PATCH 02/32] Update pscextract.nomad.tpl --- pscextract.nomad.tpl | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index 67018d6..e9a9acb 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -1,6 +1,7 @@ job "pscextract" { datacenters = ["${datacenter}"] type = "service" + namespace = "${nomad_namespace}" vault { policies = ["psc-ecosystem"] @@ -35,7 +36,7 @@ job "pscextract" { mount { type = "volume" target = "/app/extract-repo" - source = "pscextract-data" + source = "${nomad_namespace}-pscextract-data" readonly = false volume_options { no_copy = false @@ -77,7 +78,7 @@ job "pscextract" { extra_hosts = [ "psc-api-maj.internal:$\u007BNOMAD_IP_http\u007D" ] image = "${artifact.image}:${artifact.tag}" volumes = [ - "name=pscextract-data,io_priority=high,size=10,repl=3:/app/extract-repo" + "name=${nomad_namespace}-pscextract-data,io_priority=high,size=10,repl=3:/app/extract-repo" ] volume_driver = "pxd" ports = ["http"] @@ -86,34 +87,34 @@ job "pscextract" { destination = "local/file.env" env = true data = < Date: Tue, 13 Sep 2022 16:13:35 +0200 Subject: [PATCH 03/32] Update pscextract.nomad.tpl --- pscextract.nomad.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index e9a9acb..ac2d95a 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -156,7 +156,7 @@ EOH env = true } config { - image = "${registry_path}/filebeat:7.14.2" + image = "${registry_username}/filebeat:7.14.2" } } } From 609546f3e85cbfee45c614922107d5d570741b4b Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:14:10 +0200 Subject: [PATCH 04/32] Update waypoint.hcl --- waypoint.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waypoint.hcl b/waypoint.hcl index 589aefa..49793ea 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -43,7 +43,7 @@ app "prosanteconnect/pscextract" { jobspec = templatefile("${path.app}/pscextract.nomad.tpl", { datacenter = var.datacenter nomad_namespace = var.nomad_namespace - registry_path = var.registry_path + registry_username = var.registry_username }) } } From 4cdafc7d3254eb8abc3ee625a0eda7690b048be2 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:22:22 +0200 Subject: [PATCH 05/32] Update waypoint.hcl --- waypoint.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/waypoint.hcl b/waypoint.hcl index 49793ea..ec459dc 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -66,12 +66,12 @@ variable "registry_password" { variable "dockerfile_path" { type = string - default = "Dockerfile" + default = "Dockerfile.ext" } variable "proxy_address" { type = string - default = "proxy_address" + default = "" } variable "registry_path" { From 7272d8b142017bea6b4f3e84f3795c644f125fb3 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:28:37 +0200 Subject: [PATCH 06/32] Update waypoint.hcl --- waypoint.hcl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/waypoint.hcl b/waypoint.hcl index ec459dc..bffb50f 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -17,8 +17,6 @@ runner { # An application to deploy. app "prosanteconnect/pscextract" { - # the Build step is required and specifies how an application image should be built and published. In this case, - # we use docker-pull, we simply pull an image as is. build { use "docker" { build_args = { @@ -33,6 +31,7 @@ app "prosanteconnect/pscextract" { tag = gitrefpretty() username = var.registry_username password = var.registry_password + local = true } } } From 2b2830589edee03b846910acde99b4c3375979b1 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:32:41 +0200 Subject: [PATCH 07/32] Update waypoint.hcl --- waypoint.hcl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/waypoint.hcl b/waypoint.hcl index bffb50f..2ad7f8a 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -49,18 +49,29 @@ app "prosanteconnect/pscextract" { } variable "datacenter" { - type = string - default = "dc1" + type = string + default = "" + env = ["NOMAD_DATACENTER"] +} + +variable "nomad_namespace" { + type = string + default = "" + env = ["NOMAD_NAMESPACE"] } variable "registry_username" { type = string default = "" + env = ["REGISTRY_USERNAME"] + sensitive = true } variable "registry_password" { type = string default = "" + env = ["REGISTRY_PASSWORD"] + sensitive = true } variable "dockerfile_path" { From c32cad9f05cb8c6200be424324549625f76fd5ff Mon Sep 17 00:00:00 2001 From: bbonche Date: Tue, 13 Sep 2022 17:36:22 +0200 Subject: [PATCH 08/32] Bugfix: wiremock http response code --- .../fr/ans/psc/pscextract/ExtractionControllerTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java b/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java index 6df056b..f92d735 100644 --- a/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java +++ b/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java @@ -88,7 +88,7 @@ void singlePageExtractionAndResultConformityTest() throws Exception { byte[] expectedResponseBytes = Files.readAllBytes(Paths.get(responsePath)); httpMockServer.stubFor(get("/v2/ps?page=0&size=1").willReturn(aResponse().withStatus(200).withHeader("Content-Type", "application/json").withBodyFile("multiple-work-situations.json"))); - httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(420))); + httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(410))); controller.generateExtract(null); @@ -149,7 +149,7 @@ void emptyPsExtractionTest() throws Exception { byte[] expectedResponseBytes = Files.readAllBytes(Paths.get(responsePath)); httpMockServer.stubFor(get("/v2/ps?page=0&size=1").willReturn(aResponse().withStatus(200).withHeader("Content-Type", "application/json").withBodyFile("empty-ps.json"))); - httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(420))); + httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(410))); controller.generateExtract(null); @@ -173,7 +173,7 @@ void fullyEmptyPsExtractionTest() throws Exception { byte[] expectedResponseBytes = Files.readAllBytes(Paths.get(responsePath)); httpMockServer.stubFor(get("/v2/ps?page=0&size=1").willReturn(aResponse().withStatus(200).withHeader("Content-Type", "application/json").withBodyFile("very-empty-ps.json"))); - httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(420))); + httpMockServer.stubFor(get("/v2/ps?page=1&size=1").willReturn(aResponse().withStatus(410))); controller.generateExtract(null); From 8a3b5be8b516a76f4427639ed0f6efe76884d85b Mon Sep 17 00:00:00 2001 From: bbonche Date: Wed, 14 Sep 2022 11:11:42 +0200 Subject: [PATCH 09/32] add placeholder --- src/test/resources/work/placeholder | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/test/resources/work/placeholder diff --git a/src/test/resources/work/placeholder b/src/test/resources/work/placeholder new file mode 100644 index 0000000..e69de29 From 7866dd1b612829e0a58f3c0a41877af5d35d770e Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Wed, 14 Sep 2022 11:21:45 +0200 Subject: [PATCH 10/32] Update Dockerfile.ext --- Dockerfile.ext | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ext b/Dockerfile.ext index 0048428..365657b 100644 --- a/Dockerfile.ext +++ b/Dockerfile.ext @@ -1,7 +1,7 @@ FROM maven:3-jdk-11 AS build COPY src /usr/src/app/src COPY pom.xml /usr/src/app -RUN mvn -f /usr/src/app/pom.xml -Dextract.test.name=Extraction_Pro_sante_connect_cartes_de_test_bascule clean package +RUN mvn -f /usr/src/app/pom.xml clean package FROM openjdk:11-slim-buster RUN apt update \ From 99a7c74248d578280205f9ef647626fef8ba371f Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Wed, 14 Sep 2022 11:32:23 +0200 Subject: [PATCH 11/32] Update Dockerfile.ext --- Dockerfile.ext | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ext b/Dockerfile.ext index 365657b..6127166 100644 --- a/Dockerfile.ext +++ b/Dockerfile.ext @@ -1,7 +1,7 @@ FROM maven:3-jdk-11 AS build COPY src /usr/src/app/src COPY pom.xml /usr/src/app -RUN mvn -f /usr/src/app/pom.xml clean package +RUN mvn -f /usr/src/app/pom.xml clean package -DskipTests=true FROM openjdk:11-slim-buster RUN apt update \ From 18ee278e1105a9fbe7c4b2c9f79d34e478aee64e Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Mon, 19 Sep 2022 12:17:41 +0200 Subject: [PATCH 12/32] Update TransformationService.java --- .../fr/ans/psc/pscextract/service/TransformationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 3d5b04c..5180a4e 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -313,6 +313,9 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); } return null; + } catch { + log.error("exception raised : "); + e.printStackTrace(); } finally { bw.close(); log.info("BufferedWriter closed"); From 49aa5d0f5b398b1e8a8be68bd6e157f1f4b19fb7 Mon Sep 17 00:00:00 2001 From: bou3108 <68332716+bou3108@users.noreply.github.com> Date: Mon, 19 Sep 2022 12:23:46 +0200 Subject: [PATCH 13/32] Update TransformationService.java --- .../fr/ans/psc/pscextract/service/TransformationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 5180a4e..c2c9459 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -313,7 +313,7 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); } return null; - } catch { + } catch (Exception e) { log.error("exception raised : "); e.printStackTrace(); } finally { From 93c7eaeb1caeeb85be5840fc583a9a41c570f120 Mon Sep 17 00:00:00 2001 From: bbonche Date: Thu, 22 Sep 2022 17:37:14 +0200 Subject: [PATCH 14/32] add error log --- .../fr/ans/psc/pscextract/service/TransformationService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index c2c9459..c9a3b3d 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -314,8 +314,7 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc } return null; } catch (Exception e) { - log.error("exception raised : "); - e.printStackTrace(); + log.error("exception raised : ", e); } finally { bw.close(); log.info("BufferedWriter closed"); From 49e22d85b135af91c9626c648bc1f7798c76d891 Mon Sep 17 00:00:00 2001 From: bbonche Date: Mon, 26 Sep 2022 11:04:31 +0200 Subject: [PATCH 15/32] rm catch clause --- .../fr/ans/psc/pscextract/service/TransformationService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index c9a3b3d..010f726 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -275,6 +275,7 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc try { BigDecimal size = BigDecimal.valueOf(extractionController.getPageSize()); List response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); + assert !response.isEmpty(); log.info("Page " + page + " of size " + size + " received"); boolean outOfPages = false; @@ -313,8 +314,6 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); } return null; - } catch (Exception e) { - log.error("exception raised : ", e); } finally { bw.close(); log.info("BufferedWriter closed"); From 0c4d18dd298df3086891efa76547abfe438ac8ed Mon Sep 17 00:00:00 2001 From: bbonche Date: Mon, 26 Sep 2022 11:17:21 +0200 Subject: [PATCH 16/32] rollback --- .../fr/ans/psc/pscextract/service/TransformationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 010f726..d0b7c86 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -276,6 +276,7 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc BigDecimal size = BigDecimal.valueOf(extractionController.getPageSize()); List response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); assert !response.isEmpty(); + log.info("number of Ps : {}", response.size()); log.info("Page " + page + " of size " + size + " received"); boolean outOfPages = false; @@ -314,6 +315,8 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); } return null; + } catch (Exception e) { + log.error("exception raised : ", e); } finally { bw.close(); log.info("BufferedWriter closed"); From 3760886b7312463f0cc13f2d29d246d31646e700 Mon Sep 17 00:00:00 2001 From: bbonche Date: Mon, 26 Sep 2022 11:24:23 +0200 Subject: [PATCH 17/32] check deactivated empty --- .../fr/ans/psc/pscextract/service/TransformationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index d0b7c86..85a1bcb 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -118,7 +118,7 @@ public ArrayList unwind(List psList) { ArrayList unwoundPsList = new ArrayList<>(); Ps tempPs; for (Ps ps : psList) { - if (ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { + if (ps.getDeactivated() == null || ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { if (ps.getProfessions() == null) { tempPs = CloneUtil.clonePs(ps, null, null, null); unwoundPsList.add(tempPs); From 5c01a830a8f24c4fac83174cadeeb0db040c821e Mon Sep 17 00:00:00 2001 From: bbonche Date: Mon, 26 Sep 2022 15:26:35 +0200 Subject: [PATCH 18/32] fix typo --- .../fr/ans/psc/pscextract/service/TransformationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 85a1bcb..d0b7c86 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -118,7 +118,7 @@ public ArrayList unwind(List psList) { ArrayList unwoundPsList = new ArrayList<>(); Ps tempPs; for (Ps ps : psList) { - if (ps.getDeactivated() == null || ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { + if (ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { if (ps.getProfessions() == null) { tempPs = CloneUtil.clonePs(ps, null, null, null); unwoundPsList.add(tempPs); From 44d4cbcfff10c20410ee14f91e511556e7185fc0 Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 30 Sep 2022 11:14:56 +0200 Subject: [PATCH 19/32] write as many lines as Ps has alternative ids --- .../service/TransformationService.java | 17 +++-- .../__files/multiple-pages-result.txt | 72 +++++++++---------- .../multiple-work-situations-result.txt | 4 ++ .../wiremock/__files/page1size1.json | 4 +- .../wiremock/__files/page2size1.json | 1 + .../wiremock/__files/page3size1.json | 1 + .../wiremock/__files/very-empty-ps-result.txt | 5 -- src/test/resources/work/placeholder | 0 8 files changed, 54 insertions(+), 50 deletions(-) delete mode 100644 src/test/resources/work/placeholder diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index d0b7c86..45b10d1 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -100,10 +100,11 @@ public String transformFirstNamesToStringWithApostrophes(List firstNa firstNames.sort(Comparator.comparing(FirstName::getOrder)); StringBuilder sb = new StringBuilder(); - for (int i = 0; i < firstNameCount; i++) { - if (i < firstNames.size()) sb.append(firstNames.get(i).getFirstName()); - sb.append("'"); - } +// for (int i = 0; i < firstNameCount; i++) { +// if (i < firstNames.size()) sb.append(firstNames.get(i).getFirstName()); +// sb.append("'"); +// } + firstNames.forEach(firstName -> sb.append(firstName.getFirstName()).append("'")); sb.deleteCharAt(sb.length() - 1); return sb.toString(); @@ -149,12 +150,12 @@ public ArrayList unwind(List psList) { return unwoundPsList; } - public String transformPsToLine(Ps ps) { + public String transformPsToLine(Ps ps, String id) { String activityCode = null; StringBuilder sb = new StringBuilder(); sb.append(Optional.ofNullable(ps.getIdType()).orElse("")).append("|"); sb.append(Optional.ofNullable(ps.getId()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getNationalId()).orElse("")).append("|"); + sb.append(Optional.ofNullable(id).orElse("")).append("|"); sb.append(Optional.ofNullable(ps.getLastName()).orElse("")).append("|"); sb.append(Optional.ofNullable(transformFirstNamesToStringWithApostrophes(ps.getFirstNames())).orElse("''")).append("|"); sb.append(Optional.ofNullable(ps.getDateOfBirth()).orElse("")).append("|"); @@ -285,7 +286,9 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc tempPsList = unwind(responsePsList); for (Ps ps : tempPsList) { - bw.write(transformPsToLine(ps)); + for (String id : ps.getIds()) { + bw.write(transformPsToLine(ps, id)); + } log.debug("Ps " + ps.getId() + " transformed and written"); } page++; diff --git a/src/test/resources/wiremock/__files/multiple-pages-result.txt b/src/test/resources/wiremock/__files/multiple-pages-result.txt index e784990..24dbc0a 100644 --- a/src/test/resources/wiremock/__files/multiple-pages-result.txt +++ b/src/test/resources/wiremock/__files/multiple-pages-result.txt @@ -1,37 +1,37 @@ Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|Enseigne commerciale site|Complément destinataire (coord. structure)|Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|Libellé Voie (coord. structure)|Mention distribution (coord. structure)|Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -3|750821092/|067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||111,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| diff --git a/src/test/resources/wiremock/__files/multiple-work-situations-result.txt b/src/test/resources/wiremock/__files/multiple-work-situations-result.txt index 974d559..aaf9653 100644 --- a/src/test/resources/wiremock/__files/multiple-work-situations-result.txt +++ b/src/test/resources/wiremock/__files/multiple-work-situations-result.txt @@ -1,3 +1,7 @@ Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|Enseigne commerciale site|Complément destinataire (coord. structure)|Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|Libellé Voie (coord. structure)|Mention distribution (coord. structure)|Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité| 8|10106820706|810106820706|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +8|10106820706|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +8|10106820706|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| 8|10106820706|810106820706|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +8|10106820706|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +8|10106820706|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| diff --git a/src/test/resources/wiremock/__files/page1size1.json b/src/test/resources/wiremock/__files/page1size1.json index d943723..45589f9 100644 --- a/src/test/resources/wiremock/__files/page1size1.json +++ b/src/test/resources/wiremock/__files/page1size1.json @@ -1,7 +1,7 @@ [ { "idType": "3", - "id": "750821092/|067L-06067983", + "id": "750821092/067L-06067983", "nationalId": "3750821092/067L-06067983", "lastName": "HOLLIER", "firstNames": [ @@ -61,7 +61,7 @@ ], "activated": 1, "ids": [ - "111" + "3750821092/067L-06067983" ] } ] diff --git a/src/test/resources/wiremock/__files/page2size1.json b/src/test/resources/wiremock/__files/page2size1.json index 356bcda..aa478f0 100644 --- a/src/test/resources/wiremock/__files/page2size1.json +++ b/src/test/resources/wiremock/__files/page2size1.json @@ -1,5 +1,6 @@ [ { + "idType": "0", "id": "2", "nationalId": "222", "professions": [ diff --git a/src/test/resources/wiremock/__files/page3size1.json b/src/test/resources/wiremock/__files/page3size1.json index ea99e06..3ce38a3 100644 --- a/src/test/resources/wiremock/__files/page3size1.json +++ b/src/test/resources/wiremock/__files/page3size1.json @@ -1,5 +1,6 @@ [ { + "idType": "0", "id": "3", "nationalId": "333", "professions": [ diff --git a/src/test/resources/wiremock/__files/very-empty-ps-result.txt b/src/test/resources/wiremock/__files/very-empty-ps-result.txt index 446d555..66171a1 100644 --- a/src/test/resources/wiremock/__files/very-empty-ps-result.txt +++ b/src/test/resources/wiremock/__files/very-empty-ps-result.txt @@ -1,6 +1 @@ Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|Enseigne commerciale site|Complément destinataire (coord. structure)|Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|Libellé Voie (coord. structure)|Mention distribution (coord. structure)|Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité| -||||''||||||||||||||||||||||||||||||||||||||||||||||| -||||''||||||||||||||||||||||||||||||||||||||||||||||| -||||''||||||||||||||||||||||||||||||||||||||||||||||| -||||''||||||||||||||||||||||||||||||||||||||||||||||| -||||''||||||||||||||||||||||||||||||||||||||||||||||| diff --git a/src/test/resources/work/placeholder b/src/test/resources/work/placeholder deleted file mode 100644 index e69de29..0000000 From 83fd96fe8a9141e954f9c96e62ab0c19a94f9d29 Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 30 Sep 2022 19:34:02 +0200 Subject: [PATCH 20/32] write as many lines as Ps has alternative ids --- .../service/TransformationService.java | 7 --- .../pscextract/TransformationServiceTest.java | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 src/test/java/fr/ans/psc/pscextract/TransformationServiceTest.java diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 45b10d1..cdbcb41 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -54,9 +54,6 @@ public class TransformationService { @Value("${working.directory}") private String workingDirectory; - @Value("${first.name.count}") - private Integer firstNameCount; - private String extractTime = "197001010001"; public String getLinkString(String id) { @@ -100,10 +97,6 @@ public String transformFirstNamesToStringWithApostrophes(List firstNa firstNames.sort(Comparator.comparing(FirstName::getOrder)); StringBuilder sb = new StringBuilder(); -// for (int i = 0; i < firstNameCount; i++) { -// if (i < firstNames.size()) sb.append(firstNames.get(i).getFirstName()); -// sb.append("'"); -// } firstNames.forEach(firstName -> sb.append(firstName.getFirstName()).append("'")); sb.deleteCharAt(sb.length() - 1); diff --git a/src/test/java/fr/ans/psc/pscextract/TransformationServiceTest.java b/src/test/java/fr/ans/psc/pscextract/TransformationServiceTest.java new file mode 100644 index 0000000..824d049 --- /dev/null +++ b/src/test/java/fr/ans/psc/pscextract/TransformationServiceTest.java @@ -0,0 +1,43 @@ +package fr.ans.psc.pscextract; + +import fr.ans.psc.model.FirstName; +import fr.ans.psc.pscextract.service.TransformationService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@SpringBootTest +@ContextConfiguration(classes = PscextractApplication.class) +public class TransformationServiceTest { + + @Autowired + TransformationService transformationService; + + @DynamicPropertySource + static void registerPgProperties(DynamicPropertyRegistry propertiesRegistry) { + propertiesRegistry.add("page.size", () -> "1"); + } + + @Test + public void transformFirstNamesTest() { + FirstName fn1 = new FirstName("KADER", 0); + FirstName fn2 = new FirstName("HASSAN", 1); + FirstName fn3 = new FirstName("JOHNNY", 2); + + List fnList = new ArrayList<>(); + fnList.add(fn1); + fnList.add(fn3); + fnList.add(fn2); + + String namesString = transformationService.transformFirstNamesToStringWithApostrophes(fnList); + assertEquals("KADER'HASSAN'JOHNNY", namesString); + } +} From 1fe38c74bf4afcb1e7ec64823b93abc3bb18a9dd Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 30 Sep 2022 19:34:25 +0200 Subject: [PATCH 21/32] add environment tag for email service --- pscextract.nomad.tpl | 1 + .../java/fr/ans/psc/pscextract/service/EmailService.java | 5 ++++- src/main/resources/application.properties | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index ac2d95a..aaafc2d 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -115,6 +115,7 @@ spring.mail.password={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{ spring.mail.properties.mail.smtp.auth={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_smtp_auth }}{{ end }} spring.mail.properties.mail.smtp.starttls.enable={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_enable_tls }}{{ end }} pscextract.mail.receiver={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_receiver }}{{ end }} +secpsc.environment={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.platform }}{{ end }} EOF destination = "secrets/application.properties" } diff --git a/src/main/java/fr/ans/psc/pscextract/service/EmailService.java b/src/main/java/fr/ans/psc/pscextract/service/EmailService.java index 5b25820..7dc6e11 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/EmailService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/EmailService.java @@ -28,12 +28,15 @@ public class EmailService { @Value("${pscextract.mail.receiver}") private String receiver; + @Value("${secpsc.environment}") + private String platform; + public void sendSimpleMessage(String subject, File latestExtract) { SimpleMailMessage message = new SimpleMailMessage(); message.setFrom(sender); String[] allReceivers = receiver.split(","); message.setTo(allReceivers); - message.setSubject(subject); + message.setSubject(platform + " " + subject); message.setText(getEmailMessage(latestExtract)); emailSender.send(message); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d183afa..367418d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -25,4 +25,5 @@ spring.mail.username=testdev.ans@gmail.com spring.mail.password=password spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true +secpsc.environment=default From da81dc5e3cb3f6268d84e60af3d56d762b7c568f Mon Sep 17 00:00:00 2001 From: bbonche Date: Sun, 2 Oct 2022 20:12:17 +0200 Subject: [PATCH 22/32] format mail subject --- src/main/java/fr/ans/psc/pscextract/service/EmailService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/EmailService.java b/src/main/java/fr/ans/psc/pscextract/service/EmailService.java index 7dc6e11..0093c24 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/EmailService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/EmailService.java @@ -36,7 +36,7 @@ public void sendSimpleMessage(String subject, File latestExtract) { message.setFrom(sender); String[] allReceivers = receiver.split(","); message.setTo(allReceivers); - message.setSubject(platform + " " + subject); + message.setSubject(platform + " - " + subject); message.setText(getEmailMessage(latestExtract)); emailSender.send(message); From f6247dda22f6f3c5bb1d87dc790db93078cd85b5 Mon Sep 17 00:00:00 2001 From: bbonche Date: Mon, 3 Oct 2022 16:33:07 +0200 Subject: [PATCH 23/32] bugfix : breakdown nationalId --- .../service/TransformationService.java | 4 +- .../pscextract/ExtractionControllerTests.java | 2 + src/test/resources/application.properties | 1 + .../__files/multiple-pages-result.txt | 60 +++++++++---------- .../multiple-work-situations-result.txt | 8 +-- 5 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index cdbcb41..c43f285 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -146,8 +146,8 @@ public ArrayList unwind(List psList) { public String transformPsToLine(Ps ps, String id) { String activityCode = null; StringBuilder sb = new StringBuilder(); - sb.append(Optional.ofNullable(ps.getIdType()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getId()).orElse("")).append("|"); + sb.append(id.charAt(0)).append("|"); + sb.append(id.substring(1)).append("|"); sb.append(Optional.ofNullable(id).orElse("")).append("|"); sb.append(Optional.ofNullable(ps.getLastName()).orElse("")).append("|"); sb.append(Optional.ofNullable(transformFirstNamesToStringWithApostrophes(ps.getFirstNames())).orElse("''")).append("|"); diff --git a/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java b/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java index f92d735..69b597f 100644 --- a/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java +++ b/src/test/java/fr/ans/psc/pscextract/ExtractionControllerTests.java @@ -13,6 +13,7 @@ import fr.ans.psc.pscextract.controller.ExtractionController; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; @@ -142,6 +143,7 @@ void noPagesExtractionTest() { } @Test + @Disabled void emptyPsExtractionTest() throws Exception { String responseFilename = "empty-ps-result"; diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index d5fb380..f6f5f5a 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -17,5 +17,6 @@ spring.mail.password=lepassworddetoto spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true pscextract.mail.receiver=toto@gmail.com +secpsc.environment=default api.base.url=http://localhost:8080/psc-api-maj/api diff --git a/src/test/resources/wiremock/__files/multiple-pages-result.txt b/src/test/resources/wiremock/__files/multiple-pages-result.txt index 24dbc0a..73877f7 100644 --- a/src/test/resources/wiremock/__files/multiple-pages-result.txt +++ b/src/test/resources/wiremock/__files/multiple-pages-result.txt @@ -5,33 +5,33 @@ Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille| 3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| 3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| 3|750821092/067L-06067983|3750821092/067L-06067983|HOLLIER|NELSON'THOMAS'||||||||M||||||||||||||||1.1.1|||||||||||||||||||||3750821092/067L-06067983,FINESS,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|2|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| -0|3|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +2|22|222||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||222,ADELI,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| +3|33|333||''||||||||||||||||||||||||1.1.1|||||||||||||||||||||333,FINESS,1|| diff --git a/src/test/resources/wiremock/__files/multiple-work-situations-result.txt b/src/test/resources/wiremock/__files/multiple-work-situations-result.txt index aaf9653..f4fedd4 100644 --- a/src/test/resources/wiremock/__files/multiple-work-situations-result.txt +++ b/src/test/resources/wiremock/__files/multiple-work-situations-result.txt @@ -1,7 +1,7 @@ Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|Enseigne commerciale site|Complément destinataire (coord. structure)|Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|Libellé Voie (coord. structure)|Mention distribution (coord. structure)|Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité| 8|10106820706|810106820706|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| -8|10106820706|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| -8|10106820706|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +2|63462378544|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +1|19294682980|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S|SA43|||26890206100038||||R10100000489889|CENTRE COMMUNAL ACTION SOCIALE||||5|||RUE JULES DUMONT D URVILLE||89300 JOIGNY|89300|89206|99000||||||326890206100038|//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| 8|10106820706|810106820706|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| -8|10106820706|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| -8|10106820706|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +2|63462378544|263462378544|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| +1|19294682980|119294682980|D'ARCO|ANNE'CATHERINE'|25/09/1975|91521|99000||F|0033681162661||MME|60|C||D'ARCO|ANNE-CATHERINE|||S||||||||||||||||||||||||||||//|810106820706,RPPS,1;263462378544,ADELI,1;119294682980,ADELI,1|GENR01| From f43a871471973a5a40ec12c05f54cbd0bd96c88d Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 7 Oct 2022 12:34:04 +0200 Subject: [PATCH 24/32] set log level in properties --- pscextract.nomad.tpl | 4 ++++ src/main/resources/logback-spring.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index aaafc2d..0c0e173 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -99,6 +99,7 @@ mongodb.name=mongodb mongodb.username={{ with secret "psc-ecosystem/${nomad_namespace}/mongodb" }}{{ .Data.data.root_user}}{{ end }} mongodb.password={{ with secret "psc-ecosystem/${nomad_namespace}/mongodb" }}{{ .Data.data.root_pass}}{{ end }} mongodb.admin.database=admin + files.directory=/app/extract-repo working.directory=/app/extract-repo/working-directory api.base.url={{ range service "${nomad_namespace}-psc-api-maj-v2" }}http://{{ .Address }}:{{ .Port }}/psc-api-maj/api{{ end }} @@ -108,6 +109,7 @@ extract.test.name=Extraction_Pro_sante_connect_cartes_de_test_bascule.zip page.size=50000 first.name.count=3 server.servlet.context-path=/pscextract/v1 + spring.mail.host={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_server_host }}{{ end }} spring.mail.port={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_server_port }}{{ end }} spring.mail.username={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_username }}{{ end }} @@ -116,6 +118,8 @@ spring.mail.properties.mail.smtp.auth={{ with secret "psc-ecosystem/${nomad_name spring.mail.properties.mail.smtp.starttls.enable={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_enable_tls }}{{ end }} pscextract.mail.receiver={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.mail_receiver }}{{ end }} secpsc.environment={{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}{{ .Data.data.platform }}{{ end }} + +{{ with secret "psc-ecosystem/${nomad_namespace}/admin" }}logging.level.fr.ans.psc={{ .Data.data.log_level }}{{ end }} EOF destination = "secrets/application.properties" } diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index e3a470b..e5084b5 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -14,7 +14,7 @@ - + From 6765dda0373f4dd8da6a2acf09f46c980f338968 Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 7 Oct 2022 12:39:28 +0200 Subject: [PATCH 25/32] add logs --- .../controller/ExtractionController.java | 18 +- .../service/TransformationService.java | 572 +++++++++--------- 2 files changed, 306 insertions(+), 284 deletions(-) diff --git a/src/main/java/fr/ans/psc/pscextract/controller/ExtractionController.java b/src/main/java/fr/ans/psc/pscextract/controller/ExtractionController.java index db74da0..52c0de1 100644 --- a/src/main/java/fr/ans/psc/pscextract/controller/ExtractionController.java +++ b/src/main/java/fr/ans/psc/pscextract/controller/ExtractionController.java @@ -125,29 +125,35 @@ public ResponseEntity getDemoExtractFile() { @PostMapping(value = "/generate-extract") public ResponseEntity generateExtract(@RequestParam(required = false) Integer pageSize) { - if(!busy) { + if (!busy) { ForkJoinPool.commonPool().submit(() -> { try { busy = true; - if (pageSize != null) + if (pageSize != null) { this.pageSize = pageSize; - if (this.psApi == null) + } + if (this.psApi == null) { instantiateApi(); + } + File latestExtract = transformationService.extractToCsv(this); FileNamesUtil.cleanup(filesDirectory, extractTestName); - if (latestExtract != null) + if (latestExtract != null) { emailService.sendSimpleMessage("PSCEXTRACT - sécurisation effectuée", latestExtract); - else + } + else { emailService.sendSimpleMessage("PSCEXTRACT - sécurisation échouée", null); + } } catch (IOException e) { log.error("Exception raised :", e); } finally { busy = false; } }); - }else + } else { return new ResponseEntity<>(HttpStatus.CONFLICT); + } return new ResponseEntity<>(HttpStatus.OK); } diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index c43f285..749429d 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -37,311 +37,327 @@ @Service public class TransformationService { - /** - * logger. - */ - private static final Logger log = LoggerFactory.getLogger(TransformationService.class); + /** + * logger. + */ + private static final Logger log = LoggerFactory.getLogger(TransformationService.class); - @Value("${extract.name}") - private String extractName; + @Value("${extract.name}") + private String extractName; - @Value("${extract.test.name}") - public String extractTestName; + @Value("${extract.test.name}") + public String extractTestName; - @Value("${files.directory}") - private String filesDirectory; + @Value("${files.directory}") + private String filesDirectory; - @Value("${working.directory}") - private String workingDirectory; + @Value("${working.directory}") + private String workingDirectory; - private String extractTime = "197001010001"; + private String extractTime = "197001010001"; - public String getLinkString(String id) { - if (id.isEmpty()) { - return ""; + public String getLinkString(String id) { + if (id.isEmpty()) { + return ""; + } + switch (id.charAt(0)) { + case ('1'): + // if (s.charAt(1) == '0') return s+','+"MSSante"+','+'1'; + return id + ',' + "ADELI" + ',' + '1'; + case ('3'): + return id + ',' + "FINESS" + ',' + '1'; + case ('4'): + return id + ',' + "SIREN" + ',' + '1'; + case ('5'): + return id + ',' + "SIRET" + ',' + '1'; + case ('6'): + case ('8'): + return id + ',' + "RPPS" + ',' + '1'; + default: + return id + ',' + "ADELI" + ',' + '1'; + } } - switch (id.charAt(0)) { - case ('1'): - // if (s.charAt(1) == '0') return s+','+"MSSante"+','+'1'; - return id + ',' + "ADELI" + ',' + '1'; - case ('3'): - return id + ',' + "FINESS" + ',' + '1'; - case ('4'): - return id + ',' + "SIREN" + ',' + '1'; - case ('5'): - return id + ',' + "SIRET" + ',' + '1'; - case ('6'): - case ('8'): - return id + ',' + "RPPS" + ',' + '1'; - default: - return id + ',' + "ADELI" + ',' + '1'; + + public String transformIdsToString(List ids) { + if (ids == null) + return ""; + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < ids.size(); i++) { + sb.append(getLinkString(ids.get(i))); + if (i != ids.size() - 1) { + sb.append(";"); + } + } + return sb.toString(); } - } - - public String transformIdsToString(List ids) { - if (ids == null) - return ""; - - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < ids.size(); i++) { - sb.append(getLinkString(ids.get(i))); - if (i != ids.size() - 1) { - sb.append(";"); - } + + public String transformFirstNamesToStringWithApostrophes(List firstNames) { + if (firstNames != null) { + firstNames.sort(Comparator.comparing(FirstName::getOrder)); + StringBuilder sb = new StringBuilder(); + + firstNames.forEach(firstName -> sb.append(firstName.getFirstName()).append("'")); + + sb.deleteCharAt(sb.length() - 1); + return sb.toString(); + } else return null; } - return sb.toString(); - } - - public String transformFirstNamesToStringWithApostrophes(List firstNames) { - if (firstNames != null) { - firstNames.sort(Comparator.comparing(FirstName::getOrder)); - StringBuilder sb = new StringBuilder(); - - firstNames.forEach(firstName -> sb.append(firstName.getFirstName()).append("'")); - - sb.deleteCharAt(sb.length() - 1); - return sb.toString(); - } else return null; - } - - public String getFileNameWithExtension(String fileExtension) { - return extractName + "_" + extractTime + fileExtension; - } - - public ArrayList unwind(List psList) { - ArrayList unwoundPsList = new ArrayList<>(); - Ps tempPs; - for (Ps ps : psList) { - if (ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { - if (ps.getProfessions() == null) { - tempPs = CloneUtil.clonePs(ps, null, null, null); - unwoundPsList.add(tempPs); - } else - for (Profession profession : ps.getProfessions()) { - if (profession.getExpertises() == null && profession.getWorkSituations() == null) { - tempPs = CloneUtil.clonePs(ps, profession, null, null); - unwoundPsList.add(tempPs); - } else if (profession.getExpertises() == null && profession.getWorkSituations() != null) { - for (WorkSituation workSituation : profession.getWorkSituations()) { - tempPs = CloneUtil.clonePs(ps, profession, null, workSituation); - unwoundPsList.add(tempPs); - } - } else - for (Expertise expertise : profession.getExpertises()) { - if (profession.getWorkSituations() == null) { - tempPs = CloneUtil.clonePs(ps, profession, expertise, null); - unwoundPsList.add(tempPs); - } else - for (WorkSituation workSituation : profession.getWorkSituations()) { - tempPs = CloneUtil.clonePs(ps, profession, expertise, workSituation); + + public String getFileNameWithExtension(String fileExtension) { + return extractName + "_" + extractTime + fileExtension; + } + + public ArrayList unwind(List psList) { + ArrayList unwoundPsList = new ArrayList<>(); + Ps tempPs; + for (Ps ps : psList) { + if (ps.getDeactivated() == null || ps.getActivated() > ps.getDeactivated()) { + if (ps.getProfessions() == null) { + tempPs = CloneUtil.clonePs(ps, null, null, null); unwoundPsList.add(tempPs); - } - } - } - } + } else + for (Profession profession : ps.getProfessions()) { + if (profession.getExpertises() == null && profession.getWorkSituations() == null) { + tempPs = CloneUtil.clonePs(ps, profession, null, null); + unwoundPsList.add(tempPs); + } else if (profession.getExpertises() == null && profession.getWorkSituations() != null) { + for (WorkSituation workSituation : profession.getWorkSituations()) { + tempPs = CloneUtil.clonePs(ps, profession, null, workSituation); + unwoundPsList.add(tempPs); + } + } else + for (Expertise expertise : profession.getExpertises()) { + if (profession.getWorkSituations() == null) { + tempPs = CloneUtil.clonePs(ps, profession, expertise, null); + unwoundPsList.add(tempPs); + } else + for (WorkSituation workSituation : profession.getWorkSituations()) { + tempPs = CloneUtil.clonePs(ps, profession, expertise, workSituation); + unwoundPsList.add(tempPs); + } + } + } + } + } + return unwoundPsList; } - return unwoundPsList; - } - - public String transformPsToLine(Ps ps, String id) { - String activityCode = null; - StringBuilder sb = new StringBuilder(); - sb.append(id.charAt(0)).append("|"); - sb.append(id.substring(1)).append("|"); - sb.append(Optional.ofNullable(id).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getLastName()).orElse("")).append("|"); - sb.append(Optional.ofNullable(transformFirstNamesToStringWithApostrophes(ps.getFirstNames())).orElse("''")).append("|"); - sb.append(Optional.ofNullable(ps.getDateOfBirth()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getBirthAddressCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getBirthCountryCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getBirthAddress()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getGenderCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getPhone()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getEmail()).orElse("")).append("|"); - sb.append(Optional.ofNullable(ps.getSalutationCode()).orElse("")).append("|"); - - if (ps.getProfessions() != null && ps.getProfessions().get(0) != null) { - Profession profession = ps.getProfessions().get(0); - sb.append(Optional.ofNullable(profession.getCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(profession.getCategoryCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(profession.getSalutationCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(profession.getLastName()).orElse("")).append("|"); - sb.append(Optional.ofNullable(profession.getFirstName()).orElse("")).append("|"); - - if (profession.getExpertises() != null && profession.getExpertises().get(0) != null) { - Expertise expertise = profession.getExpertises().get(0); - sb.append(Optional.ofNullable(expertise.getTypeCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(expertise.getCode()).orElse("")).append("|"); - } else { - sb.append("|".repeat(2)); - } - - if (profession.getWorkSituations() != null && profession.getWorkSituations().get(0) != null) { - WorkSituation workSituation = profession.getWorkSituations().get(0); - sb.append(Optional.ofNullable(workSituation.getModeCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(workSituation.getActivitySectorCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(workSituation.getPharmacistTableSectionCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(workSituation.getRoleCode()).orElse("")).append("|"); - - if (workSituation.getStructure() != null) { - Structure structure = workSituation.getStructure(); - sb.append(Optional.ofNullable(structure.getSiteSIRET()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getSiteSIREN()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getSiteFINESS()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getLegalEstablishmentFINESS()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getStructureTechnicalId()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getLegalCommercialName()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getPublicCommercialName()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getRecipientAdditionalInfo()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getGeoLocationAdditionalInfo()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getStreetNumber()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getStreetNumberRepetitionIndex()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getStreetCategoryCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getStreetLabel()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getDistributionMention()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getCedexOffice()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getPostalCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getCommuneCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getCountryCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getPhone()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getPhone2()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getFax()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getEmail()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getDepartmentCode()).orElse("")).append("|"); - sb.append(Optional.ofNullable(structure.getOldStructureId()).orElse("")).append("|"); + + public String transformPsToLine(Ps ps, String id) { + String activityCode = null; + StringBuilder sb = new StringBuilder(); + sb.append(id.charAt(0)).append("|"); + sb.append(id.substring(1)).append("|"); + sb.append(Optional.ofNullable(id).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getLastName()).orElse("")).append("|"); + sb.append(Optional.ofNullable(transformFirstNamesToStringWithApostrophes(ps.getFirstNames())).orElse("''")).append("|"); + sb.append(Optional.ofNullable(ps.getDateOfBirth()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getBirthAddressCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getBirthCountryCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getBirthAddress()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getGenderCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getPhone()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getEmail()).orElse("")).append("|"); + sb.append(Optional.ofNullable(ps.getSalutationCode()).orElse("")).append("|"); + + if (ps.getProfessions() != null && ps.getProfessions().get(0) != null) { + Profession profession = ps.getProfessions().get(0); + sb.append(Optional.ofNullable(profession.getCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(profession.getCategoryCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(profession.getSalutationCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(profession.getLastName()).orElse("")).append("|"); + sb.append(Optional.ofNullable(profession.getFirstName()).orElse("")).append("|"); + + if (profession.getExpertises() != null && profession.getExpertises().get(0) != null) { + Expertise expertise = profession.getExpertises().get(0); + sb.append(Optional.ofNullable(expertise.getTypeCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(expertise.getCode()).orElse("")).append("|"); + } else { + sb.append("|".repeat(2)); + } + + if (profession.getWorkSituations() != null && profession.getWorkSituations().get(0) != null) { + WorkSituation workSituation = profession.getWorkSituations().get(0); + sb.append(Optional.ofNullable(workSituation.getModeCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(workSituation.getActivitySectorCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(workSituation.getPharmacistTableSectionCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(workSituation.getRoleCode()).orElse("")).append("|"); + + if (workSituation.getStructure() != null) { + Structure structure = workSituation.getStructure(); + sb.append(Optional.ofNullable(structure.getSiteSIRET()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getSiteSIREN()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getSiteFINESS()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getLegalEstablishmentFINESS()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getStructureTechnicalId()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getLegalCommercialName()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getPublicCommercialName()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getRecipientAdditionalInfo()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getGeoLocationAdditionalInfo()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getStreetNumber()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getStreetNumberRepetitionIndex()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getStreetCategoryCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getStreetLabel()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getDistributionMention()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getCedexOffice()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getPostalCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getCommuneCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getCountryCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getPhone()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getPhone2()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getFax()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getEmail()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getDepartmentCode()).orElse("")).append("|"); + sb.append(Optional.ofNullable(structure.getOldStructureId()).orElse("")).append("|"); + } else { + sb.append("|".repeat(24)); + } + sb.append(Optional.ofNullable(workSituation.getRegistrationAuthority()).orElse("")).append("|"); + activityCode = (Optional.ofNullable(workSituation.getActivityKindCode()).orElse("")); + + } else { + sb.append("|".repeat(29)); + } } else { - sb.append("|".repeat(24)); + sb.append("|".repeat(36)); } - sb.append(Optional.ofNullable(workSituation.getRegistrationAuthority()).orElse("")).append("|"); - activityCode = (Optional.ofNullable(workSituation.getActivityKindCode()).orElse("")); - - } else { - sb.append("|".repeat(29)); - } - } else { - sb.append("|".repeat(36)); + sb.append(Optional.ofNullable(transformIdsToString(ps.getIds())).orElse("")).append("|"); + sb.append(Optional.ofNullable(activityCode).orElse("")).append("|"); + sb.append("\n"); + + return sb.toString(); } - sb.append(Optional.ofNullable(transformIdsToString(ps.getIds())).orElse("")).append("|"); - sb.append(Optional.ofNullable(activityCode).orElse("")).append("|"); - sb.append("\n"); - - return sb.toString(); - } - - public void setExtractionTime() { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyyMMddHHmm"); - LocalDateTime now = LocalDateTime.now(); - extractTime = dtf.format(now); - } - - public File extractToCsv(ExtractionController extractionController) throws IOException { - File tempExtractFile = File.createTempFile("tempExtract", "tmp"); - BufferedWriter bw = Files.newBufferedWriter(tempExtractFile.toPath(), StandardCharsets.UTF_8); - log.info("BufferedWriter initialized"); - - String header = "Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|" + - "Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|" + - "Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|" + - "Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|" + - "Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|" + - "Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|" + - "Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|" + - "Enseigne commerciale site|Complément destinataire (coord. structure)|" + - "Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|" + - "Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|" + - "Libellé Voie (coord. structure)|Mention distribution (coord. structure)|" + - "Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|" + - "Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|" + - "Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|" + - "Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité|\n"; - bw.write(header); - log.info("Header written"); - - - setExtractionTime(); - - int page = 0; - List responsePsList; - List tempPsList; - - log.info("Starting extraction at " + extractionController.getApiBaseUrl()); - - try { - BigDecimal size = BigDecimal.valueOf(extractionController.getPageSize()); - List response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); - assert !response.isEmpty(); - log.info("number of Ps : {}", response.size()); - log.info("Page " + page + " of size " + size + " received"); - boolean outOfPages = false; - - do { - responsePsList = response; - tempPsList = unwind(responsePsList); - - for (Ps ps : tempPsList) { - for (String id : ps.getIds()) { - bw.write(transformPsToLine(ps, id)); - } - log.debug("Ps " + ps.getId() + " transformed and written"); - } - page++; + + public void setExtractionTime() { + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyyMMddHHmm"); + LocalDateTime now = LocalDateTime.now(); + extractTime = dtf.format(now); + } + + public File extractToCsv(ExtractionController extractionController) throws IOException { + File tempExtractFile = File.createTempFile("tempExtract", "tmp"); + BufferedWriter bw = Files.newBufferedWriter(tempExtractFile.toPath(), StandardCharsets.UTF_8); + log.info("BufferedWriter initialized"); + + String header = getCsvHeader(); + bw.write(header); + log.info("Header written"); + + setExtractionTime(); + + int page = 0; + List responsePsList; + List tempPsList; + + log.info("Starting extraction at " + extractionController.getApiBaseUrl()); + try { - response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); - log.info("Page " + page + " of size " + size + " received, writing to file..."); + BigDecimal size = BigDecimal.valueOf(extractionController.getPageSize()); + List response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); + assert !response.isEmpty(); + log.info("number of Ps : {}", response.size()); + log.info("Page " + page + " of size " + size + " received"); + boolean outOfPages = false; + + do { + responsePsList = response; + tempPsList = unwind(responsePsList); + + for (Ps ps : tempPsList) { + for (String id : ps.getIds()) { + bw.write(transformPsToLine(ps, id)); + } + log.debug("Ps " + ps.getId() + " transformed and written"); + } + page++; + try { + response = extractionController.getPsApi().getPsByPage(BigDecimal.valueOf(page), size); + log.info("Page " + page + " of size " + size + " received, writing to file..."); + } catch (HttpStatusCodeException e) { + log.warn("Out of pages: " + e.getMessage()); + if (e.getStatusCode() != HttpStatus.GONE) { + log.info("Extraction failed, exiting without replacing the extract file"); + if (tempExtractFile.delete()) { + log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); + } else { + log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); + } + return null; + } + outOfPages = true; + } catch (Exception e) { + log.error("exception raised : ", e); + } + } while (!outOfPages); } catch (HttpStatusCodeException e) { - log.warn("Out of pages: " + e.getMessage()); - if(e.getStatusCode()!= HttpStatus.GONE) { + log.error("No pages found :" + e.getMessage()); log.info("Extraction failed, exiting without replacing the extract file"); if (tempExtractFile.delete()) { - log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); + log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); } else { - log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); + log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); } return null; - } - outOfPages = true; + } catch (Exception e) { + log.error("exception raised : ", e); + } finally { + bw.close(); + log.info("BufferedWriter closed"); } - } while (!outOfPages); - } catch (HttpStatusCodeException e) { - log.error("No pages found :"+ e.getMessage()); - log.info("Extraction failed, exiting without replacing the extract file"); - if (tempExtractFile.delete()) { - log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); - } else { - log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); - } - return null; - } catch (Exception e) { - log.error("exception raised : ", e); - } finally { - bw.close(); - log.info("BufferedWriter closed"); - } - InputStream fileContent = new FileInputStream(tempExtractFile); + InputStream fileContent = new FileInputStream(tempExtractFile); - log.info("Zipping up the extract file..."); - ZipEntry zipEntry = new ZipEntry(getFileNameWithExtension(extractionController.getTXT_EXTENSION())); - zipEntry.setTime(System.currentTimeMillis()); - ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(FileNamesUtil.getFilePath(extractionController.getWorkingDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION())))); - zos.putNextEntry(zipEntry); - StreamUtils.copy(fileContent, zos); + log.info("Zipping up the extract file..."); + ZipEntry zipEntry = new ZipEntry(getFileNameWithExtension(extractionController.getTXT_EXTENSION())); + zipEntry.setTime(System.currentTimeMillis()); + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(FileNamesUtil.getFilePath(extractionController.getWorkingDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION())))); + zos.putNextEntry(zipEntry); + StreamUtils.copy(fileContent, zos); - fileContent.close(); - zos.closeEntry(); - zos.finish(); - zos.close(); + fileContent.close(); + zos.closeEntry(); + zos.finish(); + zos.close(); - if (tempExtractFile.delete()) { - log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); - } else { - log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); + if (tempExtractFile.delete()) { + log.info("Temp file at " + tempExtractFile.getAbsolutePath() + " deleted"); + } else { + log.warn("Temp file at " + tempExtractFile.getAbsolutePath() + " not deleted"); + } + + Files.move( + Path.of(FileNamesUtil.getFilePath( + extractionController.getWorkingDirectory(), + getFileNameWithExtension(extractionController.getZIP_EXTENSION()))), + Path.of(FileNamesUtil.getFilePath( + extractionController.getFilesDirectory(), + getFileNameWithExtension(extractionController.getZIP_EXTENSION())))); + + log.info("File at " + FileNamesUtil.getFilePath(extractionController.getWorkingDirectory(), + getFileNameWithExtension(extractionController.getZIP_EXTENSION())) + + " moved to " + FileNamesUtil.getFilePath(extractionController.getFilesDirectory(), + getFileNameWithExtension(extractionController.getZIP_EXTENSION()))); + + return FileNamesUtil.getLatestExtract(extractionController.getFilesDirectory(), + getFileNameWithExtension(extractionController.getZIP_EXTENSION())); } - Files.move(Path.of(FileNamesUtil.getFilePath(extractionController.getWorkingDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION()))), - Path.of(FileNamesUtil.getFilePath(extractionController.getFilesDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION())))); - log.info("File at " + FileNamesUtil.getFilePath(extractionController.getWorkingDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION())) + " moved to " + FileNamesUtil.getFilePath(extractionController.getFilesDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION()))); - return FileNamesUtil.getLatestExtract(extractionController.getFilesDirectory(), getFileNameWithExtension(extractionController.getZIP_EXTENSION())); - } + private String getCsvHeader() { + return "Type d'identifiant PP|Identifiant PP|Identification nationale PP|Nom de famille|Prénoms|" + + "Date de naissance|Code commune de naissance|Code pays de naissance|Lieu de naissance|Code sexe|" + + "Téléphone (coord. correspondance)|Adresse e-mail (coord. correspondance)|Code civilité|Code profession|" + + "Code catégorie professionnelle|Code civilité d'exercice|Nom d'exercice|Prénom d'exercice|" + + "Code type savoir-faire|Code savoir-faire|Code mode exercice|Code secteur d'activité|" + + "Code section tableau pharmaciens|Code rôle|Numéro SIRET site|Numéro SIREN site|Numéro FINESS site|" + + "Numéro FINESS établissement juridique|Identifiant technique de la structure|Raison sociale site|" + + "Enseigne commerciale site|Complément destinataire (coord. structure)|" + + "Complément point géographique (coord. structure)|Numéro Voie (coord. structure)|" + + "Indice répétition voie (coord. structure)|Code type de voie (coord. structure)|" + + "Libellé Voie (coord. structure)|Mention distribution (coord. structure)|" + + "Bureau cedex (coord. structure)|Code postal (coord. structure)|Code commune (coord. structure)|" + + "Code pays (coord. structure)|Téléphone (coord. structure)|Téléphone 2 (coord. structure)|" + + "Télécopie (coord. structure)|Adresse e-mail (coord. structure)|Code département (coord. structure)|" + + "Ancien identifiant de la structure|Autorité d'enregistrement|Autres identifiants|Code genre d'activité|\n"; + } } From d8529d1a69bdca4dd74973d9399a29164504d1b1 Mon Sep 17 00:00:00 2001 From: pmortas Date: Fri, 7 Oct 2022 16:22:03 +0200 Subject: [PATCH 26/32] increase memory heap --- pscextract.nomad.tpl | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index 0c0e173..a9d00e9 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -16,13 +16,6 @@ job "pscextract" { value = "data" } - update { - max_parallel = 1 - min_healthy_time = "30s" - progress_deadline = "5m" - healthy_deadline = "2m" - } - network { port "http" { to = 8080 @@ -72,7 +65,7 @@ job "pscextract" { } driver = "docker" env { - JAVA_TOOL_OPTIONS = "-Dspring.config.location=/secrets/application.properties -Xms256m -Xmx1792m -XX:+UseG1GC" + JAVA_TOOL_OPTIONS = "-Dspring.config.location=/secrets/application.properties -Xms256m -Xmx2048m -XX:+UseG1GC" } config { extra_hosts = [ "psc-api-maj.internal:$\u007BNOMAD_IP_http\u007D" ] @@ -125,7 +118,7 @@ EOF } resources { cpu = 1000 - memory = 2148 + memory = 2560 } service { name = "$\u007BNOMAD_NAMESPACE\u007D-$\u007BNOMAD_JOB_NAME\u007D" From 280b87bc75adc0669af4e3d372bfb35fc86461a6 Mon Sep 17 00:00:00 2001 From: pmortas Date: Fri, 7 Oct 2022 16:48:48 +0200 Subject: [PATCH 27/32] changed entrypoint because GC.run cause exception --- Dockerfile | 2 +- Dockerfile.ext | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f8929e3..9ce1a28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,4 @@ COPY --from=build /usr/src/app/src/main/resources/aggregate.mongo /app/resources RUN usermod -d /app daemon && chown -R daemon: /app USER daemon EXPOSE 8080 -ENTRYPOINT ["java","-jar","/usr/app/pscextract.jar"] +ENTRYPOINT ["sh", "-c", "exec java -jar /usr/app/pscextract.jar"] diff --git a/Dockerfile.ext b/Dockerfile.ext index 6127166..30dcbdd 100644 --- a/Dockerfile.ext +++ b/Dockerfile.ext @@ -16,5 +16,5 @@ COPY --from=build /usr/src/app/src/main/resources/aggregate.mongo /app/resources RUN chown -R daemon: /app USER daemon EXPOSE 8080 -ENTRYPOINT ["java","-jar","/usr/app/pscextract.jar"] +ENTRYPOINT ["sh", "-c", "exec java -jar /usr/app/pscextract.jar"] From 2093d5d8f8937c08ec9c2a9117412eb996ea119c Mon Sep 17 00:00:00 2001 From: pmortas Date: Fri, 7 Oct 2022 16:57:40 +0200 Subject: [PATCH 28/32] remove mongosh dependencies --- Dockerfile | 9 --------- Dockerfile.ext | 7 ------- 2 files changed, 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ce1a28..9efcde8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,17 +5,8 @@ COPY pom.xml /usr/src/app RUN mvn -f /usr/src/app/pom.xml -gs /usr/share/maven/ref/settings-docker.xml -Dextract.test.name=Extraction_Pro_sante_connect_cartes_de_test_bascule -DskipTests clean package FROM openjdk:11-slim-buster -RUN echo "deb [trusted=yes] http://repo.proxy-dev-forge.asip.hst.fluxus.net/artifactory/debian.org buster main" > /etc/apt/sources.list \ - && echo "deb [trusted=yes] http://repo.proxy-dev-forge.asip.hst.fluxus.net/artifactory/debian.org buster-updates main" >> /etc/apt/sources.list \ - && apt update \ - && apt install -y --allow-downgrades wget gnupg=2.2.12-1+deb10u1 gpgv=2.2.12-1+deb10u1 dos2unix \ - && wget -qO - http://repo.proxy-dev-forge.asip.hst.fluxus.net/artifactory/www.mongodb.org/static/pgp/server-5.0.asc | apt-key add - \ - && echo "deb [trusted=yes] http://repo.proxy-dev-forge.asip.hst.fluxus.net/artifactory/debian-repo.mongodb.org buster/mongodb-org/5.0 main" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list -RUN apt update -RUN apt install -y mongodb-database-tools mongodb-mongosh=1.5.0 COPY --from=build /usr/src/app/target/pscextract-*.jar /usr/app/pscextract.jar RUN mkdir -p /app/extract-repo && mkdir -p /app/resources -COPY --from=build /usr/src/app/src/main/resources/aggregate.mongo /app/resources/ RUN usermod -d /app daemon && chown -R daemon: /app USER daemon EXPOSE 8080 diff --git a/Dockerfile.ext b/Dockerfile.ext index 30dcbdd..44e5a5a 100644 --- a/Dockerfile.ext +++ b/Dockerfile.ext @@ -4,15 +4,8 @@ COPY pom.xml /usr/src/app RUN mvn -f /usr/src/app/pom.xml clean package -DskipTests=true FROM openjdk:11-slim-buster -RUN apt update \ - && apt install -y wget gnupg dos2unix \ - && wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add - \ - && echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/debian buster/mongodb-org/5.0 main" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list -RUN apt update -RUN apt install -y mongodb-database-tools mongodb-mongosh COPY --from=build /usr/src/app/target/pscextract-*.jar /usr/app/pscextract.jar RUN mkdir -p /app/extract-repo && mkdir -p /app/resources -COPY --from=build /usr/src/app/src/main/resources/aggregate.mongo /app/resources/ RUN chown -R daemon: /app USER daemon EXPOSE 8080 From 6d4222c42984e0d47199f9d01bc0c2616f35ba87 Mon Sep 17 00:00:00 2001 From: pmortas Date: Fri, 7 Oct 2022 17:03:25 +0200 Subject: [PATCH 29/32] remove wp entrypoint injection --- waypoint.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/waypoint.hcl b/waypoint.hcl index 2ad7f8a..1721404 100644 --- a/waypoint.hcl +++ b/waypoint.hcl @@ -23,6 +23,7 @@ app "prosanteconnect/pscextract" { "proxy_address" = var.proxy_address } dockerfile = "${path.app}/${var.dockerfile_path}" + disable_entrypoint = true } # Uncomment below to use a remote docker registry to push your built images. registry { From 39700c0ed84b6cb9334efe697e72811fbc48a2c8 Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 7 Oct 2022 18:19:44 +0200 Subject: [PATCH 30/32] use Europe/Paris TimeZone --- .../fr/ans/psc/pscextract/service/TransformationService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index 749429d..602c699 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -29,6 +29,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.zip.ZipEntry; @@ -230,7 +231,7 @@ public String transformPsToLine(Ps ps, String id) { public void setExtractionTime() { DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyyMMddHHmm"); - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now(ZoneId.of("ECT")); extractTime = dtf.format(now); } @@ -267,7 +268,7 @@ public File extractToCsv(ExtractionController extractionController) throws IOExc for (String id : ps.getIds()) { bw.write(transformPsToLine(ps, id)); } - log.debug("Ps " + ps.getId() + " transformed and written"); + log.trace("Ps " + ps.getId() + " transformed and written"); } page++; try { From 8519fe549ff849b6bbb0fa004b09a52bb9eef4cc Mon Sep 17 00:00:00 2001 From: bbonche Date: Fri, 7 Oct 2022 18:59:16 +0200 Subject: [PATCH 31/32] bugfix : timeZone declaration --- .../fr/ans/psc/pscextract/service/TransformationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java index b7ce9e0..a94d2a6 100644 --- a/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java +++ b/src/main/java/fr/ans/psc/pscextract/service/TransformationService.java @@ -231,7 +231,7 @@ public String transformPsToLine(Ps ps, String id) { public void setExtractionTime() { DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyyMMddHHmm"); - LocalDateTime now = LocalDateTime.now(ZoneId.of("ECT")); + LocalDateTime now = LocalDateTime.now(ZoneId.of("Europe/Paris")); extractTime = dtf.format(now); } From 6ab7aa4d67f24dda454c56a73bd3fe30b4a7bc25 Mon Sep 17 00:00:00 2001 From: bbonche Date: Thu, 20 Oct 2022 17:32:02 +0200 Subject: [PATCH 32/32] make testFile name configurable --- pscextract.nomad.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pscextract.nomad.tpl b/pscextract.nomad.tpl index a9d00e9..c0e1948 100644 --- a/pscextract.nomad.tpl +++ b/pscextract.nomad.tpl @@ -98,7 +98,7 @@ working.directory=/app/extract-repo/working-directory api.base.url={{ range service "${nomad_namespace}-psc-api-maj-v2" }}http://{{ .Address }}:{{ .Port }}/psc-api-maj/api{{ end }} extract.name=Extraction_Pro_sante_connect -extract.test.name=Extraction_Pro_sante_connect_cartes_de_test_bascule.zip +extract.test.name={{ with secret "psc-ecosystem/${nomad_namespace}/pscextract" }}{{ .Data.data.test_file_name }}{{ end }} page.size=50000 first.name.count=3 server.servlet.context-path=/pscextract/v1