From 45da8f17458a8ad4e3ab3eb4a84430d2ba9d3d85 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 14 Jul 2017 10:44:00 +0200 Subject: [PATCH 01/68] Updated to version 2.2.1-SNAPSHOT. --- demetra-dotstat-core/pom.xml | 2 +- demetra-dotstat-desktop/pom.xml | 2 +- pom.xml | 2 +- sdmx-facade/pom.xml | 2 +- sdmx-facade/sdmx-facade-api/pom.xml | 2 +- sdmx-facade/sdmx-facade-connectors/pom.xml | 2 +- sdmx-facade/sdmx-facade-file/pom.xml | 2 +- sdmx-facade/sdmx-facade-samples/pom.xml | 2 +- sdmx-facade/sdmx-facade-util/pom.xml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml index 0ea89b28..dc928d62 100644 --- a/demetra-dotstat-core/pom.xml +++ b/demetra-dotstat-core/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-core - 2.2.0 + 2.2.1-SNAPSHOT jar Demetra - DotStat - Core diff --git a/demetra-dotstat-desktop/pom.xml b/demetra-dotstat-desktop/pom.xml index b5ecd2ac..c7e6b4ac 100644 --- a/demetra-dotstat-desktop/pom.xml +++ b/demetra-dotstat-desktop/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-desktop - 2.2.0 + 2.2.1-SNAPSHOT nbm Demetra - DotStat - Desktop diff --git a/pom.xml b/pom.xml index 19c3a787..362e860e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-aggregator - 2.2.0 + 2.2.1-SNAPSHOT pom Demetra - DotStat diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index 4558c08d..429935d1 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -4,7 +4,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT pom diff --git a/sdmx-facade/sdmx-facade-api/pom.xml b/sdmx-facade/sdmx-facade-api/pom.xml index 5b65140f..60df5f78 100644 --- a/sdmx-facade/sdmx-facade-api/pom.xml +++ b/sdmx-facade/sdmx-facade-api/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT sdmx-facade-api diff --git a/sdmx-facade/sdmx-facade-connectors/pom.xml b/sdmx-facade/sdmx-facade-connectors/pom.xml index 240b8962..2f7ea596 100644 --- a/sdmx-facade/sdmx-facade-connectors/pom.xml +++ b/sdmx-facade/sdmx-facade-connectors/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT sdmx-facade-connectors diff --git a/sdmx-facade/sdmx-facade-file/pom.xml b/sdmx-facade/sdmx-facade-file/pom.xml index dcec1a04..072146fa 100644 --- a/sdmx-facade/sdmx-facade-file/pom.xml +++ b/sdmx-facade/sdmx-facade-file/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT sdmx-facade-file diff --git a/sdmx-facade/sdmx-facade-samples/pom.xml b/sdmx-facade/sdmx-facade-samples/pom.xml index 316a9ef0..d0c2bb8a 100644 --- a/sdmx-facade/sdmx-facade-samples/pom.xml +++ b/sdmx-facade/sdmx-facade-samples/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT sdmx-facade-samples diff --git a/sdmx-facade/sdmx-facade-util/pom.xml b/sdmx-facade/sdmx-facade-util/pom.xml index e911a219..e7d6bcf8 100644 --- a/sdmx-facade/sdmx-facade-util/pom.xml +++ b/sdmx-facade/sdmx-facade-util/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.0 + 2.2.1-SNAPSHOT sdmx-facade-util From 281f66f20df71f2b3699a2f5e2152406e0c77d6b Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 16 Oct 2017 16:10:09 +0200 Subject: [PATCH 02/68] Fixed joinup repository. --- .travis.yml | 1 - demetra-dotstat-core/pom.xml | 6 +++--- demetra-dotstat-desktop/pom.xml | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 070ad913..21f8717e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: java jdk: - oraclejdk8 -sudo: required cache: directories: diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml index dc928d62..3f9d024d 100644 --- a/demetra-dotstat-core/pom.xml +++ b/demetra-dotstat-core/pom.xml @@ -18,7 +18,7 @@ European Union Public Licence (EUPL) - https://joinup.ec.europa.eu/software/page/eupl/licence-eupl + https://joinup.ec.europa.eu/page/eupl-text-11-12 @@ -115,8 +115,8 @@ http://bits.netbeans.org/maven2/ - joinup-releases - https://joinup.ec.europa.eu/nexus/content/repositories/releases/ + oss-jfrog-artifactory-releases + https://oss.jfrog.org/artifactory/oss-release-local \ No newline at end of file diff --git a/demetra-dotstat-desktop/pom.xml b/demetra-dotstat-desktop/pom.xml index c7e6b4ac..6a9d6f7d 100644 --- a/demetra-dotstat-desktop/pom.xml +++ b/demetra-dotstat-desktop/pom.xml @@ -18,7 +18,7 @@ European Union Public Licence (EUPL) - https://joinup.ec.europa.eu/software/page/eupl/licence-eupl + https://joinup.ec.europa.eu/page/eupl-text-11-12 @@ -138,8 +138,8 @@ http://bits.netbeans.org/maven2/ - joinup-releases - https://joinup.ec.europa.eu/nexus/content/repositories/releases/ + oss-jfrog-artifactory-releases + https://oss.jfrog.org/artifactory/oss-release-local From 26365e1a760658af4f16baac25d09bf5d7575167 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 16 Oct 2017 16:53:19 +0200 Subject: [PATCH 03/68] Updated dependencies. --- sdmx-facade/pom.xml | 11 ++++++++--- .../java/be/nbb/sdmx/facade/repo/SdmxRepository.java | 5 ++--- .../be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java | 6 ++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index 429935d1..aa9f29b8 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -21,10 +21,10 @@ 1.8 1.8 - 3.0.1 - 1.16.12 + 3.0.2 + 1.16.18 4.12 - 3.6.2 + 3.8.0 @@ -66,6 +66,11 @@ sdmx-facade-connectors ${project.version} + + ${project.groupId} + sdmx-facade-file + ${project.version} + ${project.groupId} sdmx-facade-samples diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index c275cba0..268203ba 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -57,7 +57,8 @@ public class SdmxRepository { @lombok.NonNull Map> data; - boolean seriesKeysOnlySupported; + @lombok.Builder.Default + boolean seriesKeysOnlySupported = true; @Nonnull public SdmxConnection asConnection() { @@ -70,8 +71,6 @@ public SdmxConnection asConnection() { public static final class Builder { - private boolean seriesKeysOnlySupported = true; - @Nonnull public Builder clearData() { if (this.data == null) { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index 77025faf..1390949f 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.tck.DataCursorAssert; import java.time.LocalDateTime; import java.util.Collections; +import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; /** @@ -41,6 +42,11 @@ public void testDataCursorCompliance() { DataCursorAssert.assertCompliance(() -> Series.asCursor(Collections.singletonList(series), Key.ALL)); } + @Test + public void testBuilder() { + assertThat(SdmxRepository.builder().name("test").data(xyz, series).build().isSeriesKeysOnlySupported()).isTrue(); + } + private final DataflowRef xyz = DataflowRef.parse("XYZ"); private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); private final SdmxRepository repo = SdmxRepository.builder().name("test").data(xyz, series).build(); From c25beba9bdd999cdb4badf516f5f267d34f25855 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 16 Oct 2017 17:26:13 +0200 Subject: [PATCH 04/68] Refactored ResourceRef as an interface. --- .../be/nbb/sdmx/facade/DataStructureRef.java | 24 ++++- .../java/be/nbb/sdmx/facade/DataflowRef.java | 30 ++++-- .../java/be/nbb/sdmx/facade/ResourceRef.java | 92 +------------------ .../main/java/internal/util/ResourceRefs.java | 75 +++++++++++++++ 4 files changed, 120 insertions(+), 101 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java index 32d903f3..de713f7a 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java @@ -16,27 +16,41 @@ */ package be.nbb.sdmx.facade; +import internal.util.ResourceRefs; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.AccessLevel; /** * Identifier of a data structure. * * @author Philippe Charles */ -public final class DataStructureRef extends ResourceRef { +@lombok.Value +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +public final class DataStructureRef implements ResourceRef { - private DataStructureRef(String agencyId, String id, String version) { - super(agencyId, id, version); + @lombok.NonNull + private String agencyId; + + @lombok.NonNull + private String id; + + @lombok.NonNull + private String version; + + @Override + public String toString() { + return ResourceRefs.toString(this); } @Nonnull public static DataStructureRef parse(@Nonnull String input) throws IllegalArgumentException { - return ResourceRef.parse(input, DataStructureRef::new); + return ResourceRefs.parse(input, DataStructureRef::new); } @Nonnull public static DataStructureRef of(@Nullable String agencyId, @Nonnull String flowId, @Nullable String version) throws IllegalArgumentException { - return ResourceRef.of(agencyId, flowId, version, DataStructureRef::new); + return ResourceRefs.of(agencyId, flowId, version, DataStructureRef::new); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java index 6602543e..8d1a7fe6 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java @@ -16,9 +16,10 @@ */ package be.nbb.sdmx.facade; +import internal.util.ResourceRefs; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.annotation.concurrent.Immutable; +import lombok.AccessLevel; /** * Identifier of a data flow used in a data (or meta data) query. @@ -34,24 +35,37 @@ * * @author Philippe Charles */ -@Immutable -public final class DataflowRef extends ResourceRef { +@lombok.Value +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +public final class DataflowRef implements ResourceRef { + + @lombok.NonNull + private String agencyId; + + @lombok.NonNull + private String id; + + @lombok.NonNull + private String version; public boolean contains(@Nonnull DataflowRef that) { - return super.contains(that); + return (this.agencyId.equals(ALL_AGENCIES) || this.agencyId.equals(that.agencyId)) + && (this.id.equals(that.id)) + && (this.version.equals(LATEST_VERSION) || this.version.equals(that.version)); } - private DataflowRef(String agencyId, String flowId, String version) { - super(agencyId, flowId, version); + @Override + public String toString() { + return ResourceRefs.toString(this); } @Nonnull public static DataflowRef parse(@Nonnull String input) throws IllegalArgumentException { - return ResourceRef.parse(input, DataflowRef::new); + return ResourceRefs.parse(input, DataflowRef::new); } @Nonnull public static DataflowRef of(@Nullable String agencyId, @Nonnull String flowId, @Nullable String version) throws IllegalArgumentException { - return ResourceRef.of(agencyId, flowId, version, DataflowRef::new); + return ResourceRefs.of(agencyId, flowId, version, DataflowRef::new); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java index 80b6d427..703ded03 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java @@ -16,108 +16,24 @@ */ package be.nbb.sdmx.facade; -import java.util.Objects; import javax.annotation.Nonnull; -import javax.annotation.Nullable; /** * Abstract identifier of a resource. * * @author Philippe Charles */ -public abstract class ResourceRef { +public interface ResourceRef { public static final String ALL_AGENCIES = "all"; public static final String LATEST_VERSION = "latest"; - private final String agencyId; - private final String id; - private final String version; - - protected ResourceRef(@Nonnull String agencyId, @Nonnull String id, @Nonnull String version) { - this.agencyId = agencyId; - this.id = id; - this.version = version; - } - @Nonnull - public String getAgencyId() { - return agencyId; - } + String getAgencyId(); @Nonnull - public String getId() { - return id; - } - - @Nonnull - public String getVersion() { - return version; - } - - protected boolean contains(@Nonnull ResourceRef that) { - return (this.agencyId.equals(ALL_AGENCIES) || this.agencyId.equals(that.agencyId)) - && (this.id.equals(that.id)) - && (this.version.equals(LATEST_VERSION) || this.version.equals(that.version)); - } - - @Override - public String toString() { - return agencyId + "," + id + "," + version; - } - - @Override - public int hashCode() { - return Objects.hash(agencyId, id, version); - } - - @Override - public boolean equals(Object obj) { - return this == obj || (obj instanceof ResourceRef && equals((ResourceRef) obj)); - } - - private boolean equals(ResourceRef that) { - return this.agencyId.equals(that.agencyId) - && this.id.equals(that.id) - && this.version.equals(that.version); - } + String getId(); @Nonnull - private static String emptyToDefault(@Nonnull String input, @Nonnull String defaultValue) { - return input.isEmpty() ? defaultValue : input; - } - - @Nonnull - private static String nullOrEmptyToDefault(@Nullable String input, @Nonnull String defaultValue) { - return input == null || input.isEmpty() ? defaultValue : input; - } - - @Nonnull - protected static T parse(@Nonnull String input, @Nonnull RefFactory factory) throws IllegalArgumentException { - String[] items = input.split(",", -1); - switch (items.length) { - case 3: - return factory.create(emptyToDefault(items[0], ALL_AGENCIES), items[1], emptyToDefault(items[2], LATEST_VERSION)); - case 2: - return factory.create(emptyToDefault(items[0], ALL_AGENCIES), items[1], LATEST_VERSION); - case 1: - return factory.create(ALL_AGENCIES, items[0], LATEST_VERSION); - default: - throw new IllegalArgumentException(input); - } - } - - @Nonnull - protected static T of(@Nullable String agencyId, @Nonnull String id, @Nullable String version, @Nonnull RefFactory factory) throws IllegalArgumentException { - if (id.contains(",")) { - throw new IllegalArgumentException(id); - } - return factory.create(nullOrEmptyToDefault(agencyId, ALL_AGENCIES), id, nullOrEmptyToDefault(version, LATEST_VERSION)); - } - - protected interface RefFactory { - - @Nonnull - T create(@Nonnull String agencyId, @Nonnull String id, @Nonnull String version); - } + String getVersion(); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java b/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java new file mode 100644 index 00000000..6020384a --- /dev/null +++ b/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util; + +import be.nbb.sdmx.facade.ResourceRef; +import static be.nbb.sdmx.facade.ResourceRef.ALL_AGENCIES; +import static be.nbb.sdmx.facade.ResourceRef.LATEST_VERSION; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class ResourceRefs { + + @Nonnull + private static String emptyToDefault(@Nonnull String input, @Nonnull String defaultValue) { + return input.isEmpty() ? defaultValue : input; + } + + @Nonnull + private static String nullOrEmptyToDefault(@Nullable String input, @Nonnull String defaultValue) { + return input == null || input.isEmpty() ? defaultValue : input; + } + + @Nonnull + public static String toString(ResourceRef ref) { + return ref.getAgencyId() + "," + ref.getId() + "," + ref.getVersion(); + } + + @Nonnull + public static T parse(@Nonnull String input, @Nonnull RefFactory factory) throws IllegalArgumentException { + String[] items = input.split(",", -1); + switch (items.length) { + case 3: + return factory.create(emptyToDefault(items[0], ALL_AGENCIES), items[1], emptyToDefault(items[2], LATEST_VERSION)); + case 2: + return factory.create(emptyToDefault(items[0], ALL_AGENCIES), items[1], LATEST_VERSION); + case 1: + return factory.create(ALL_AGENCIES, items[0], LATEST_VERSION); + default: + throw new IllegalArgumentException(input); + } + } + + @Nonnull + public static T of(@Nullable String agencyId, @Nonnull String id, @Nullable String version, @Nonnull RefFactory factory) throws IllegalArgumentException { + if (id.contains(",")) { + throw new IllegalArgumentException(id); + } + return factory.create(nullOrEmptyToDefault(agencyId, ALL_AGENCIES), id, nullOrEmptyToDefault(version, LATEST_VERSION)); + } + + public interface RefFactory { + + @Nonnull + T create(@Nonnull String agencyId, @Nonnull String id, @Nonnull String version); + } +} From c1305e850fff3779617943990f609a373d8c67a5 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 17 Oct 2017 10:36:01 +0200 Subject: [PATCH 05/68] Moved SdmxDriver to web project. --- .../nbb/demetra/dotstat/DotStatProvider.java | 6 +- .../demetra/sdmx/file/SdmxFileProvider.java | 4 +- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 6 +- .../demetra/dotstat/DotStatAccessorTest.java | 9 +- .../demetra/sdmx/web/SdmxWebProviderTest.java | 8 +- .../be/nbb/demetra/dotstat/DotStatPanel.java | 14 +-- .../demetra/dotstat/DotStatProviderBuddy.java | 9 -- .../dotstat/SdmxWsAutoCompletionService.java | 4 +- .../sdmx/web/SdmxWebProviderBuddy.java | 26 ++++-- .../internal/sdmx/SdmxAutoCompletion.java | 14 +-- .../nbb/sdmx/facade/connectors/AbsDriver.java | 10 +-- .../facade/connectors/EurostatDriver.java | 10 +-- .../nbb/sdmx/facade/connectors/IloDriver.java | 10 +-- .../nbb/sdmx/facade/connectors/ImfDriver.java | 10 +-- .../sdmx/facade/connectors/InseeDriver.java | 10 +-- .../nbb/sdmx/facade/connectors/NbbDriver.java | 10 +-- .../sdmx/facade/connectors/OecdDriver.java | 10 +-- .../sdmx/facade/connectors/Sdmx20Driver.java | 10 +-- .../sdmx/facade/connectors/Sdmx21Driver.java | 16 ++-- .../facade/connectors/SdmxDriverSupport.java | 6 +- .../nbb/sdmx/facade/connectors/UisDriver.java | 10 +-- .../nbb/sdmx/facade/connectors/WbDriver.java | 10 +-- .../nbb/sdmx/facade/web/SdmxWebDriver.java} | 8 +- .../nbb/sdmx/facade/web/SdmxWebManager.java} | 43 ++++------ .../nbb/sdmx/facade/web/WebEntryPoint.java} | 4 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 7 -- .../facade/repo/SdmxRepositoryDriver.java | 85 ------------------- .../facade/repo/SdmxRepositoryManager.java | 45 ++++++++++ 28 files changed, 180 insertions(+), 234 deletions(-) rename sdmx-facade/{sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriver.java => sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java} (85%) rename sdmx-facade/{sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriverManager.java => sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java} (62%) rename sdmx-facade/{sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/WsEntryPoint.java => sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java} (91%) delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryDriver.java create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java index bbb75e02..397a8d13 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; +import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.collect.Maps; import ec.tss.ITsProvider; import ec.tss.TsAsyncMode; @@ -55,7 +55,7 @@ public final class DotStatProvider extends DbProvider implements Ha public DotStatProvider() { super(LoggerFactory.getLogger(DotStatProvider.class), NAME, TsAsyncMode.Once); - this.connectionSupplier = new AtomicReference<>(SdmxDriverManager.getDefault()); + this.connectionSupplier = new AtomicReference<>(SdmxWebManager.ofServiceLoader()); this.languages = new AtomicReference<>(LanguagePriorityList.ANY); this.displayCodes = false; } @@ -145,7 +145,7 @@ public SdmxConnectionSupplier getConnectionSupplier() { @Override public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxDriverManager.getDefault())) { + if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxWebManager.ofServiceLoader())) { clearCache(); } } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 34ec348b..4b08124b 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -80,7 +80,7 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private final ITsProvider tsSupport; public SdmxFileProvider() { - this.connectionSupplier = new AtomicReference<>(SdmxFileManager.getDefault()); + this.connectionSupplier = new AtomicReference<>(new SdmxFileManager()); this.languages = new AtomicReference<>(LanguagePriorityList.ANY); Logger logger = LoggerFactory.getLogger(NAME); @@ -113,7 +113,7 @@ public SdmxConnectionSupplier getConnectionSupplier() { @Override public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxFileManager.getDefault())) { + if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : new SdmxFileManager())) { clearCache(); } } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index c4dd3d08..87eceb8f 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -21,7 +21,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; +import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.cache.Cache; import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; @@ -76,7 +76,7 @@ public final class SdmxWebProvider implements IDataSourceLoader, HasSdmxProperti private final ITsProvider tsSupport; public SdmxWebProvider() { - this.connectionSupplier = new AtomicReference<>(SdmxDriverManager.getDefault()); + this.connectionSupplier = new AtomicReference<>(SdmxWebManager.ofServiceLoader()); this.languages = new AtomicReference<>(LanguagePriorityList.ANY); this.displayCodes = new AtomicBoolean(false); @@ -104,7 +104,7 @@ public SdmxConnectionSupplier getConnectionSupplier() { @Override public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxDriverManager.getDefault())) { + if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxWebManager.ofServiceLoader())) { clearCache(); } } diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index f468e775..1729c0e8 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -23,9 +23,9 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.connectors.TestResource; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; -import be.nbb.sdmx.facade.repo.SdmxRepositoryDriver; +import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import com.google.common.base.Joiner; import ec.tss.tsproviders.db.DbAccessor; import ec.tss.tsproviders.db.DbSeries; @@ -44,11 +44,10 @@ */ public class DotStatAccessorTest { - private final SdmxDriverManager supplier = SdmxDriverManager.of(SdmxRepositoryDriver.builder() - .prefix("") + private final SdmxConnectionSupplier supplier = SdmxRepositoryManager.builder() .repository(TestResource.nbb()) .repository(TestResource.ecb()) - .build()); + .build(); private static DotStatBean nbbBean() { DotStatBean result = new DotStatBean(); diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java index f539edb9..a013593d 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java @@ -24,10 +24,10 @@ import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.repo.Obs; -import be.nbb.sdmx.facade.repo.SdmxRepositoryDriver; +import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import be.nbb.sdmx.facade.repo.Series; import ec.tss.TsMoniker; import ec.tss.tsproviders.DataSet; @@ -110,8 +110,8 @@ private static DataSource getSampleDataSource(DotStatProvider o) { return o.encodeBean(result); } - private static SdmxDriverManager getCustomSupplier() { - return SdmxDriverManager.of(SdmxRepositoryDriver.of(getCustomRepo())); + private static SdmxConnectionSupplier getCustomSupplier() { + return SdmxRepositoryManager.builder().repository(getCustomRepo()).build(); } private static SdmxRepository getCustomRepo() { diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java index 0bdc417e..3b22cd6a 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java @@ -19,8 +19,8 @@ import be.nbb.demetra.sdmx.web.SdmxWebProvider; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebManager; +import be.nbb.sdmx.facade.web.WebEntryPoint; import ec.nbdemetra.ui.completion.JAutoCompletionService; import ec.nbdemetra.ui.nodes.AbstractNodeBuilder; import ec.nbdemetra.ui.properties.NodePropertySetBuilder; @@ -203,9 +203,9 @@ private static Optional lookupProvider() { } private void loadEntryPoints(SdmxConnectionSupplier supplier) { - if (supplier instanceof SdmxDriverManager) { + if (supplier instanceof SdmxWebManager) { AbstractNodeBuilder b = new AbstractNodeBuilder(); - ((SdmxDriverManager) supplier).getEntryPoints().forEach(x -> b.add(new ConfigNode(x))); + ((SdmxWebManager) supplier).getEntryPoints().forEach(x -> b.add(new ConfigNode(x))); em.setRootContext(b.name("hello").build()); } } @@ -247,7 +247,7 @@ boolean valid() { private static final class ConfigNode extends AbstractNode implements Editable { - public ConfigNode(WsEntryPoint o) { + public ConfigNode(WebEntryPoint o) { super(Children.LEAF, Lookups.singleton(o)); setDisplayName(o.getName()); setShortDescription(o.getDescription()); @@ -265,7 +265,7 @@ public Image getOpenedIcon(int type) { @Override protected Sheet createSheet() { - WsEntryPoint bean = getLookup().lookup(WsEntryPoint.class); + WebEntryPoint bean = getLookup().lookup(WebEntryPoint.class); Sheet result = new Sheet(); NodePropertySetBuilder b = new NodePropertySetBuilder(); b.with(String.class) @@ -297,7 +297,7 @@ public void actionPerformed(ActionEvent e) { @Override public void edit() { if (new PropertySheetDialogBuilder().title("Edit entry point").icon(getIcon(BeanInfo.ICON_MONO_16x16)).editNode(this)) { - setDisplayName(getLookup().lookup(WsEntryPoint.class).getName()); + setDisplayName(getLookup().lookup(WebEntryPoint.class).getName()); } } } diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatProviderBuddy.java index 1217d60b..8d68e0e2 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatProviderBuddy.java @@ -16,7 +16,6 @@ */ package be.nbb.demetra.dotstat; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; import com.google.common.base.Converter; import ec.nbdemetra.db.DbProviderBuddy; import ec.nbdemetra.ui.BeanHandler; @@ -36,7 +35,6 @@ import org.netbeans.api.options.OptionsDisplayer; import org.openide.nodes.Sheet; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.util.HasCache; import ec.tstoolkit.utilities.GuavaCaches; import internal.sdmx.SdmxAutoCompletion; import java.time.Duration; @@ -61,13 +59,6 @@ public DotStatProviderBuddy() { this.tableRenderer = SdmxAutoCompletion.getFlowsRenderer(); this.columnRenderer = SdmxAutoCompletion.getDimensionsRenderer(); this.autoCompletionCache = GuavaCaches.ttlCacheAsMap(Duration.ofMinutes(1)); - initDriverCache(GuavaCaches.softValuesCacheAsMap()); - } - - private static void initDriverCache(ConcurrentMap cache) { - SdmxDriverManager.getDefault().getDrivers().stream() - .filter(o -> (o instanceof HasCache)) - .forEach(o -> ((HasCache) o).setCache(cache)); } @Override diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/SdmxWsAutoCompletionService.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/SdmxWsAutoCompletionService.java index f411695b..544478f1 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/SdmxWsAutoCompletionService.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/SdmxWsAutoCompletionService.java @@ -16,7 +16,7 @@ */ package be.nbb.demetra.dotstat; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; +import be.nbb.demetra.sdmx.web.SdmxWebProviderBuddy; import ec.nbdemetra.ui.completion.JAutoCompletionService; import ec.util.completion.AutoCompletionSource; import ec.util.completion.swing.JAutoCompletion; @@ -34,7 +34,7 @@ public final class SdmxWsAutoCompletionService extends JAutoCompletionService { public static final String PATH = "JAutoCompletionService/SdmxWs"; - private final AutoCompletionSource source = SdmxAutoCompletion.onEntryPoints(SdmxDriverManager.getDefault()); + private final AutoCompletionSource source = SdmxAutoCompletion.onEntryPoints(SdmxWebProviderBuddy.getDefaultManager()); private final ListCellRenderer renderer = SdmxAutoCompletion.getEntryPointsRenderer(); @Override diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java index f42faa6b..4bc84da3 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java @@ -21,8 +21,9 @@ import be.nbb.demetra.dotstat.SdmxWsAutoCompletionService; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; +import be.nbb.sdmx.facade.web.SdmxWebManager; import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.web.SdmxWebDriver; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import ec.nbdemetra.db.DbIcon; @@ -49,6 +50,7 @@ import org.netbeans.api.options.OptionsDisplayer; import org.openide.nodes.Sheet; import org.openide.util.ImageUtilities; +import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; @@ -60,13 +62,27 @@ @ServiceProvider(service = IDataSourceProviderBuddy.class, supersedes = "be.nbb.demetra.dotstat.DotStatProviderBuddy") public final class SdmxWebProviderBuddy implements IDataSourceProviderBuddy, IConfigurable { + private static final SdmxWebManager WEB_MANAGER = createManager(); + + public static SdmxWebManager getDefaultManager() { + return WEB_MANAGER; + } + + private static SdmxWebManager createManager() { + SdmxWebManager result = SdmxWebManager.of(Lookup.getDefault().lookupAll(SdmxWebDriver.class)); + ConcurrentMap cache = GuavaCaches.softValuesCacheAsMap(); + result.getDrivers().stream() + .filter(HasCache.class::isInstance) + .forEach(o -> ((HasCache) o).setCache(cache)); + return result; + } + private final Configurator configurator; private final ConcurrentMap autoCompletionCache; public SdmxWebProviderBuddy() { this.configurator = createConfigurator(); this.autoCompletionCache = GuavaCaches.ttlCacheAsMap(Duration.ofMinutes(1)); - initDriverCache(GuavaCaches.softValuesCacheAsMap()); } @Override @@ -161,12 +177,6 @@ public void storeBean(SdmxWebProviderBuddy resource, BuddyConfig bean) { } } - private static void initDriverCache(ConcurrentMap cache) { - SdmxDriverManager.getDefault().getDrivers().stream() - .filter(o -> (o instanceof HasCache)) - .forEach(o -> ((HasCache) o).setCache(cache)); - } - @NbBundle.Messages({ "bean.cache.description=Mechanism used to improve performance."}) private static Sheet createSheet(SdmxWebBean bean, SdmxConnectionSupplier supplier, LanguagePriorityList languages, ConcurrentMap cache) { diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java index f162d0fc..abb4283a 100644 --- a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java @@ -22,8 +22,8 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.driver.SdmxDriverManager; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebManager; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.UnexpectedIOException; import com.google.common.base.Strings; import ec.util.completion.AutoCompletionSource; @@ -49,17 +49,17 @@ @lombok.experimental.UtilityClass public class SdmxAutoCompletion { - public AutoCompletionSource onEntryPoints(SdmxDriverManager manager) { + public AutoCompletionSource onEntryPoints(SdmxWebManager manager) { return ExtAutoCompletionSource .builder(o -> manager.getEntryPoints()) .behavior(AutoCompletionSource.Behavior.SYNC) .postProcessor(SdmxAutoCompletion::filterAndSortEntryPoints) - .valueToString(WsEntryPoint::getName) + .valueToString(WebEntryPoint::getName) .build(); } public ListCellRenderer getEntryPointsRenderer() { - return CustomListCellRenderer.of(WsEntryPoint::getDescription, WsEntryPoint::getName); + return CustomListCellRenderer.of(WebEntryPoint::getDescription, WebEntryPoint::getName); } public AutoCompletionSource onFlows(SdmxConnectionSupplier supplier, LanguagePriorityList languages, Supplier source, ConcurrentMap cache) { @@ -103,11 +103,11 @@ public String getDefaultDimensionsAsString(SdmxConnectionSupplier supplier, Lang .collect(Collectors.joining(delimiter)); } - private List filterAndSortEntryPoints(List allValues, String term) { + private List filterAndSortEntryPoints(List allValues, String term) { Predicate filter = ExtAutoCompletionSource.basicFilter(term); return allValues.stream() .filter(o -> filter.test(o.getDescription()) || filter.test(o.getUri().toString())) - .sorted(Comparator.comparing(WsEntryPoint::getDescription)) + .sorted(Comparator.comparing(WebEntryPoint::getDescription)) .collect(Collectors.toList()); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java index d9050336..8c412ef3 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ABS; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class AbsDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class AbsDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:abs:"; @@ -36,7 +36,7 @@ public final class AbsDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ABS.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java index 7712b1d3..5ce10440 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.EUROSTAT; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class EurostatDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class EurostatDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:eurostat:"; @@ -36,7 +36,7 @@ public final class EurostatDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, EUROSTAT.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java index 94bfdadf..f8ed1237 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ILO; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class IloDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class IloDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:ilo:"; @@ -36,7 +36,7 @@ public final class IloDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ILO.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java index e0149b37..86ca1e5f 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.IMF; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class ImfDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class ImfDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:imf:"; @@ -36,7 +36,7 @@ public final class ImfDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, IMF.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java index 310bc8fa..3c7a3c15 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java @@ -26,8 +26,7 @@ import static be.nbb.sdmx.facade.Frequency.UNDEFINED; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.FreqParser; import be.nbb.sdmx.facade.util.HasCache; @@ -53,13 +52,14 @@ import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class InseeDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class InseeDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:insee:"; @@ -69,7 +69,7 @@ public final class InseeDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l, xml)); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java index fa0db2c7..e21ef6fb 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.NBB; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class NbbDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class NbbDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:nbb:"; @@ -36,7 +36,7 @@ public final class NbbDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, NBB.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java index 6efb40e8..eb238e38 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.OECD; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class OecdDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class OecdDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:oecd:"; @@ -36,7 +36,7 @@ public final class OecdDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, OECD.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java index df9208e8..1a134941 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java @@ -18,8 +18,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import static be.nbb.sdmx.facade.connectors.Util.NEEDS_CREDENTIALS; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.RestSdmx20Client; import java.net.URL; @@ -27,13 +26,14 @@ import java.util.List; import java.util.Map; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class Sdmx20Driver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class Sdmx20Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx20:"; @@ -41,7 +41,7 @@ public final class Sdmx20Driver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, Sdmx20Client::new); @Override - public List getDefaultEntryPoints() { + public List getDefaultEntryPoints() { return Collections.emptyList(); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java index e4b54dc7..8f713f24 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java @@ -20,8 +20,7 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.SdmxMediaType; @@ -41,13 +40,14 @@ import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class Sdmx21Driver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx21:"; @@ -57,9 +57,9 @@ public final class Sdmx21Driver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l, xml)); @Override - public List getDefaultEntryPoints() { + public List getDefaultEntryPoints() { Sdmx21EntryPointBuilder b = new Sdmx21EntryPointBuilder(); - List result = new ArrayList<>(); + List result = new ArrayList<>(); result.add(b.clear() .name("ECB") .description("European Central Bank") @@ -157,8 +157,8 @@ private Map toProperties() { return result; } - public WsEntryPoint build() { - return WsEntryPoint.builder() + public WebEntryPoint build() { + return WebEntryPoint.builder() .name(name) .description(description) .uri(PREFIX + endpoint) diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java index bb225911..e11b24bb 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CACHE_TTL; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CONNECT_TIMEOUT; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.READ_TIMEOUT; @@ -95,8 +95,8 @@ public void setCache(ConcurrentMap cache) { } @Nonnull - public static Collection entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { - return Collections.singleton(WsEntryPoint.builder().name(name).description(description).uri(url).build()); + public static Collection entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { + return Collections.singleton(WebEntryPoint.builder().name(name).description(description).uri(url).build()); } // diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java index 6803e1a7..4ac82a96 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.UIS; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class UisDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class UisDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:uis:"; @@ -36,7 +36,7 @@ public final class UisDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, UIS.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java index 9c2a6084..1e6f16e5 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java @@ -16,19 +16,19 @@ */ package be.nbb.sdmx.facade.connectors; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; +import be.nbb.sdmx.facade.web.WebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.WB; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.SdmxWebDriver; /** * * @author Philippe Charles */ -@ServiceProvider(service = SdmxDriver.class) -public final class WbDriver implements SdmxDriver, HasCache { +@ServiceProvider(service = SdmxWebDriver.class) +public final class WbDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:wb:"; @@ -36,7 +36,7 @@ public final class WbDriver implements SdmxDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, WB.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("WB", "World Bank", "sdmx:wb:http://api.worldbank.org"); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java similarity index 85% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java index 6e824de3..b9aa656d 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.driver; +package be.nbb.sdmx.facade.web; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; @@ -30,13 +30,13 @@ * @author Philippe Charles */ @ThreadSafe -public interface SdmxDriver { +public interface SdmxWebDriver { @Nonnull - SdmxConnection connect(@Nonnull URI uri, @Nonnull Map info, @Nonnull LanguagePriorityList languages) throws IOException; + SdmxConnection connect(@Nonnull URI uri, @Nonnull Map properties, @Nonnull LanguagePriorityList languages) throws IOException; boolean acceptsURI(@Nonnull URI uri) throws IOException; @Nonnull - Collection getDefaultEntryPoints(); + Collection getDefaultEntryPoints(); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriverManager.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java similarity index 62% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriverManager.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 7b24cdac..6b52d06f 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/SdmxDriverManager.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.driver; +package be.nbb.sdmx.facade.web; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; @@ -35,46 +35,46 @@ * * @author Philippe Charles */ -public final class SdmxDriverManager implements SdmxConnectionSupplier { +public final class SdmxWebManager implements SdmxConnectionSupplier { @Nonnull - public static SdmxDriverManager getDefault() { - return Holder.INSTANCE; + public static SdmxWebManager ofServiceLoader() { + return of(ServiceLoader.load(SdmxWebDriver.class)); } @Nonnull - public static SdmxDriverManager of(@Nonnull SdmxDriver... drivers) { + public static SdmxWebManager of(@Nonnull SdmxWebDriver... drivers) { return of(Arrays.asList(drivers)); } @Nonnull - public static SdmxDriverManager of(@Nonnull Iterable drivers) { - CopyOnWriteArrayList driverList = new CopyOnWriteArrayList<>(); - ConcurrentMap entryPointByName = new ConcurrentHashMap<>(); + public static SdmxWebManager of(@Nonnull Iterable drivers) { + CopyOnWriteArrayList driverList = new CopyOnWriteArrayList<>(); + ConcurrentMap entryPointByName = new ConcurrentHashMap<>(); drivers.forEach(o -> { driverList.add(o); o.getDefaultEntryPoints().forEach(x -> entryPointByName.put(x.getName(), x)); }); - return new SdmxDriverManager(driverList, entryPointByName); + return new SdmxWebManager(driverList, entryPointByName); } - private final CopyOnWriteArrayList drivers; - private final ConcurrentMap entryPointByName; + private final CopyOnWriteArrayList drivers; + private final ConcurrentMap entryPointByName; - private SdmxDriverManager(CopyOnWriteArrayList drivers, ConcurrentMap entryPointByName) { + private SdmxWebManager(CopyOnWriteArrayList drivers, ConcurrentMap entryPointByName) { this.drivers = drivers; this.entryPointByName = entryPointByName; } @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { - WsEntryPoint wsEntryPoint = entryPointByName.get(name); + WebEntryPoint wsEntryPoint = entryPointByName.get(name); if (wsEntryPoint == null) { throw new IOException("Cannot find entry point for '" + name + "'"); } URI uri = wsEntryPoint.getUri(); try { - for (SdmxDriver o : drivers) { + for (SdmxWebDriver o : drivers) { if (o.acceptsURI(uri)) { return o.connect(uri, wsEntryPoint.getProperties(), languages); } @@ -86,29 +86,22 @@ public SdmxConnection getConnection(String name, LanguagePriorityList languages) } @Nonnull - public List getDrivers() { + public List getDrivers() { return Collections.unmodifiableList(drivers); } - public void setDrivers(@Nonnull List list) { + public void setDrivers(@Nonnull List list) { drivers.clear(); drivers.addAll(list); } @Nonnull - public List getEntryPoints() { + public List getEntryPoints() { return new ArrayList<>(entryPointByName.values()); } - public void setEntryPoints(@Nonnull List list) { + public void setEntryPoints(@Nonnull List list) { entryPointByName.clear(); list.forEach(o -> entryPointByName.put(o.getName(), o)); } - - // - private static final class Holder { - - private static final SdmxDriverManager INSTANCE = of(ServiceLoader.load(SdmxDriver.class)); - } - // } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/WsEntryPoint.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java similarity index 91% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/WsEntryPoint.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java index 2587222e..96264eb9 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/driver/WsEntryPoint.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.driver; +package be.nbb.sdmx.facade.web; import java.net.URI; import java.util.Map; @@ -26,7 +26,7 @@ */ @lombok.Value @lombok.Builder(builderClassName = "Builder") -public final class WsEntryPoint { +public final class WebEntryPoint { @Nonnull String name; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 4ec32566..85e02c2d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -67,11 +67,4 @@ public ConcurrentMap getCache() { public void setCache(ConcurrentMap cache) { this.cache.set(cache != null ? cache : new ConcurrentHashMap()); } - - @Nonnull - public static SdmxFileManager getDefault() { - return INSTANCE; - } - - private static final SdmxFileManager INSTANCE = new SdmxFileManager(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryDriver.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryDriver.java deleted file mode 100644 index b91be537..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryDriver.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2016 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.repo; - -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.driver.SdmxDriver; -import be.nbb.sdmx.facade.driver.WsEntryPoint; -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -@lombok.Value -@lombok.Builder(builderClassName = "Builder") -public class SdmxRepositoryDriver implements SdmxDriver { - - @Nonnull - public static SdmxRepositoryDriver of(@Nonnull SdmxRepository repo) { - return new SdmxRepositoryDriver("sdmx:repo:", Collections.singletonList(repo)); - } - - @lombok.NonNull - String prefix; - - @lombok.NonNull - @lombok.Singular - List repositories; - - @Override - public SdmxConnection connect(URI uri, Map info, LanguagePriorityList languages) throws IOException { - return connect(getName(uri)); - } - - @Override - public boolean acceptsURI(URI uri) throws IOException { - return uri.toString().startsWith(prefix); - } - - @Override - public Collection getDefaultEntryPoints() { - return repositories.stream() - .map(o -> WsEntryPoint.builder().name(o.getName()).description("").uri(prefix + o.getName()).build()) - .collect(Collectors.toList()); - } - - private String getName(URI uri) throws IOException { - String result = uri.toString(); - int index = result.indexOf(prefix); - if (index == -1) { - throw new IOException("Invalid URI '" + uri + "'"); - } - return result.substring(prefix.length()); - } - - private SdmxConnection connect(String name) throws IOException { - return repositories.stream() - .filter(o -> o.getName().equals(name)) - .findFirst() - .orElseThrow(() -> new IOException("Cannot find '" + name + "'")) - .asConnection(); - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java new file mode 100644 index 00000000..b522e407 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java @@ -0,0 +1,45 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.repo; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import java.io.IOException; +import java.util.List; + +/** + * + * @author Philippe Charles + */ +@lombok.Value +@lombok.Builder(builderClassName = "Builder") +public final class SdmxRepositoryManager implements SdmxConnectionSupplier { + + @lombok.NonNull + @lombok.Singular + List repositories; + + @Override + public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + return repositories.stream() + .filter(o -> o.getName().equals(name)) + .findFirst() + .orElseThrow(() -> new IOException("Cannot find '" + name + "'")) + .asConnection(); + } +} From fc809b5cb7dd570921aaacb3625c21773035a0f9 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 17 Oct 2017 13:58:07 +0200 Subject: [PATCH 06/68] Added initial support of query parameters. --- .../java/internal/sdmx/SdmxQueryUtil.java | 3 +- .../demetra/dotstat/DotStatAccessorTest.java | 3 +- .../java/be/nbb/sdmx/facade/QueryDetail.java | 26 +++++++++++++++ .../be/nbb/sdmx/facade/QueryParameters.java | 33 +++++++++++++++++++ .../be/nbb/sdmx/facade/SdmxConnection.java | 2 +- .../connectors/CachedSdmxConnection.java | 3 +- .../connectors/SdmxConnectionAdapter.java | 5 ++- .../sdmx/facade/file/FileSdmxConnection.java | 6 ++-- .../facade/file/FileSdmxConnectionTest.java | 3 +- .../nbb/sdmx/facade/tck/ConnectionAssert.java | 7 ++-- .../nbb/sdmx/facade/repo/SdmxRepository.java | 3 +- 11 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java create mode 100644 sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index 8cd28051..493bcda2 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -20,6 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; import com.google.common.collect.ImmutableList; import ec.tss.tsproviders.cursor.TsCursor; @@ -87,7 +88,7 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re private final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { - return new SdmxDataAdapter(key, conn.getData(flowRef, key, seriesKeysOnly), labelAttribute); + return new SdmxDataAdapter(key, conn.getData(flowRef, key, !seriesKeysOnly ? QueryParameters.FULL : QueryParameters.SERIES_KEYS_ONLY), labelAttribute); } private TsCursor computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index 1729c0e8..dce9360f 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; @@ -99,7 +100,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { - try (DataCursor o = supplier.getConnection("NBB", LanguagePriorityList.ANY).getData(DataflowRef.of("NBB", "TEST_DATASET", null), Key.ALL, true)) { + try (DataCursor o = supplier.getConnection("NBB", LanguagePriorityList.ANY).getData(DataflowRef.of("NBB", "TEST_DATASET", null), Key.ALL, QueryParameters.SERIES_KEYS_ONLY)) { o.nextSeries(); assertThat(o.getSeriesKey()).isEqualTo(Key.parse("LOCSTL04.AUS.M")); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java new file mode 100644 index 00000000..ef14ce2d --- /dev/null +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade; + +/** + * + * @author Philippe Charles + */ +public enum QueryDetail { + + FULL, SERIES_KEYS_ONLY +} diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java new file mode 100644 index 00000000..056659de --- /dev/null +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade; + +/** + * + * @author Philippe Charles + */ +@lombok.Value +@lombok.Builder(builderClassName = "Builder") +public class QueryParameters { + + public static final QueryParameters FULL = builder().detail(QueryDetail.FULL).build(); + public static final QueryParameters SERIES_KEYS_ONLY = builder().detail(QueryDetail.SERIES_KEYS_ONLY).build(); + + @lombok.NonNull + @lombok.Builder.Default + private QueryDetail detail = QueryDetail.FULL; +} diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java index a60e280c..1eec2d14 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java @@ -39,7 +39,7 @@ public interface SdmxConnection extends Closeable { DataStructure getDataStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull Key key, boolean serieskeysonly) throws IOException; + DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull QueryParameters queryParams) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java index 1de36c89..19e29c79 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java @@ -20,6 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; @@ -108,7 +109,7 @@ protected DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeyson } cache.put(id, result); } - return result.asConnection().getData(flowRef, key, true); + return result.asConnection().getData(flowRef, key, QueryParameters.SERIES_KEYS_ONLY); } return super.loadData(flowRef, key, serieskeysonly); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java index 44cbecbd..2c40a0b0 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java @@ -21,6 +21,8 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.QueryDetail; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.util.NoOpCursor; import be.nbb.sdmx.facade.util.ObsParser; @@ -73,8 +75,9 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + final public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { checkState(); + boolean serieskeysonly = queryParams.getDetail().equals(QueryDetail.SERIES_KEYS_ONLY); if (serieskeysonly && !isSeriesKeysOnlySupported()) { throw new IllegalStateException("serieskeysonly not supported"); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java index 56abc5c3..c108d079 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java @@ -23,6 +23,8 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.QueryDetail; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; @@ -80,12 +82,12 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + final public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { checkState(); Objects.requireNonNull(flowRef); Objects.requireNonNull(key); checkFlowRef(flowRef); - return loadData(decode(), flowRef, key, serieskeysonly); + return loadData(decode(), flowRef, key, queryParams.getDetail().equals(QueryDetail.SERIES_KEYS_ONLY)); } @Override diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java index 63671805..1eb60516 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java @@ -21,6 +21,7 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; import java.io.File; @@ -56,7 +57,7 @@ public void testCompactData21() throws IOException { Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getData(file.getDataflowRef(), Key.ALL, false)) { + try (DataCursor o = conn.getData(file.getDataflowRef(), Key.ALL, QueryParameters.FULL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 55660363..15c513af 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -19,6 +19,7 @@ import internal.io.ConsumerWithIO; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; import java.io.IOException; import java.util.concurrent.Callable; @@ -51,7 +52,7 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getData(ref, Key.ALL, false), "getData(DataFlowRef, Key, boolean)"); + assertState(s, supplier, o -> o.getData(ref, Key.ALL, QueryParameters.FULL), "getData(DataFlowRef, Key, boolean)"); assertState(s, supplier, o -> o.getDataStructure(ref), "getDataStructure(DataFlowRef)"); assertState(s, supplier, o -> o.getDataflow(ref), "getDataflow(DataFlowRef)"); assertState(s, supplier, SdmxConnection::getDataflows, "getDataflows()"); @@ -59,11 +60,11 @@ public static void assertCompliance(SoftAssertions s, Callable s @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getData(null, Key.ALL, false)) + s.assertThatThrownBy(() -> conn.getData(null, Key.ALL, QueryParameters.FULL)) .as("Expecting 'getData(DataFlowRef, Key, boolean)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getData(ref, null, false)) + s.assertThatThrownBy(() -> conn.getData(ref, null, QueryParameters.FULL)) .as("Expecting 'getData(DataFlowRef, Key, boolean)' to raise NPE when called with null key") .isInstanceOf(NullPointerException.class); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 268203ba..4e74b515 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; import java.io.IOException; import java.util.ArrayList; @@ -150,7 +151,7 @@ public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { checkState(); Objects.requireNonNull(flowRef); Objects.requireNonNull(key); From 562576b0b026fd5c96b09d2e8793235239877d38 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 17 Oct 2017 14:43:41 +0200 Subject: [PATCH 07/68] Refactored connector project as web project. --- .gitignore | 3 +- demetra-dotstat-core/pom.xml | 2 +- .../demetra/dotstat/DotStatAccessorTest.java | 2 +- .../connectors/TestResource.java | 2 +- .../java/internal/sdmx/SdmxQueryUtilTest.java | 2 +- .../be/nbb/demetra/dotstat/DotStatPanel.java | 8 +- .../sdmx/web/SdmxWebProviderBuddy.java | 8 +- .../internal/sdmx/SdmxAutoCompletion.java | 10 +- sdmx-facade/pom.xml | 4 +- .../nbb/sdmx/facade/web/SdmxWebManager.java | 107 ----------- .../pom.xml | 3 +- .../sdmx/facade/web/SdmxWebEntryPoint.java} | 2 +- .../nbb/sdmx/facade/web/SdmxWebManager.java | 167 ++++++++++++++++++ .../sdmx/facade/web/spi}/SdmxWebDriver.java | 5 +- .../java/internal}/connectors/AbsDriver.java | 8 +- .../connectors/CachedSdmxConnection.java | 2 +- .../internal}/connectors/ClientSupplier.java | 2 +- .../connectors/DataCursorAdapter.java | 2 +- .../internal}/connectors/EurostatDriver.java | 8 +- .../internal}/connectors/HasDataCursor.java | 2 +- .../HasSeriesKeysOnlySupported.java | 2 +- .../java/internal}/connectors/IloDriver.java | 8 +- .../java/internal}/connectors/ImfDriver.java | 8 +- .../internal}/connectors/InseeDriver.java | 8 +- .../java/internal}/connectors/NbbDriver.java | 8 +- .../java/internal}/connectors/OecdDriver.java | 8 +- .../internal}/connectors/Sdmx20Driver.java | 10 +- .../internal}/connectors/Sdmx21Config.java | 2 +- .../internal}/connectors/Sdmx21Driver.java | 14 +- .../connectors/SdmxConnectionAdapter.java | 2 +- .../connectors/SdmxDriverSupport.java | 8 +- .../java/internal}/connectors/UisDriver.java | 8 +- .../main/java/internal}/connectors/Util.java | 2 +- .../java/internal}/connectors/WbDriver.java | 8 +- .../connectors/ConnectorsResource.java | 2 +- .../connectors/DataCursorAdapterTest.java | 2 +- .../internal}/connectors/FacadeResource.java | 2 +- .../internal}/connectors/ParsersTest.java | 2 +- .../connectors/SdmxConnectionAdapterTest.java | 2 +- 39 files changed, 257 insertions(+), 198 deletions(-) rename demetra-dotstat-core/src/test/java/{be/nbb/sdmx/facade => internal}/connectors/TestResource.java (97%) delete mode 100644 sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java rename sdmx-facade/{sdmx-facade-connectors => sdmx-facade-web}/pom.xml (94%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java => sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java} (93%) create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web => sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi}/SdmxWebDriver.java (89%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/AbsDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/CachedSdmxConnection.java (99%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/ClientSupplier.java (97%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/DataCursorAdapter.java (95%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/EurostatDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/HasDataCursor.java (96%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/HasSeriesKeysOnlySupported.java (95%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/IloDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/ImfDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/InseeDriver.java (97%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/NbbDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/OecdDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/Sdmx20Driver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/Sdmx21Config.java (97%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/Sdmx21Driver.java (95%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/SdmxConnectionAdapter.java (99%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/SdmxDriverSupport.java (92%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/UisDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/Util.java (98%) rename sdmx-facade/{sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade => sdmx-facade-web/src/main/java/internal}/connectors/WbDriver.java (87%) rename sdmx-facade/{sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade => sdmx-facade-web/src/test/java/internal}/connectors/ConnectorsResource.java (97%) rename sdmx-facade/{sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade => sdmx-facade-web/src/test/java/internal}/connectors/DataCursorAdapterTest.java (98%) rename sdmx-facade/{sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade => sdmx-facade-web/src/test/java/internal}/connectors/FacadeResource.java (99%) rename sdmx-facade/{sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade => sdmx-facade-web/src/test/java/internal}/connectors/ParsersTest.java (96%) rename sdmx-facade/{sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade => sdmx-facade-web/src/test/java/internal}/connectors/SdmxConnectionAdapterTest.java (96%) diff --git a/.gitignore b/.gitignore index 10c03ab0..51f9d064 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ /sdmx-facade/sdmx-facade-connectors/target/ /sdmx-facade/sdmx-facade-util/target/ /sdmx-facade/sdmx-facade-file/target/ -/sdmx-facade/sdmx-facade-samples/target/ \ No newline at end of file +/sdmx-facade/sdmx-facade-samples/target/ +/sdmx-facade/sdmx-facade-web/target/ \ No newline at end of file diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml index 3f9d024d..9f92e7dd 100644 --- a/demetra-dotstat-core/pom.xml +++ b/demetra-dotstat-core/pom.xml @@ -73,7 +73,7 @@ be.nbb.sdmx - sdmx-facade-connectors + sdmx-facade-web ${project.version} diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index dce9360f..a660fcbd 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -25,7 +25,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.connectors.TestResource; +import internal.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import com.google.common.base.Joiner; import ec.tss.tsproviders.db.DbAccessor; diff --git a/demetra-dotstat-core/src/test/java/be/nbb/sdmx/facade/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java similarity index 97% rename from demetra-dotstat-core/src/test/java/be/nbb/sdmx/facade/connectors/TestResource.java rename to demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 1e80b88c..19f08470 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/sdmx/facade/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.DataCursor; diff --git a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java index c7cd2d19..51a5e965 100644 --- a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java +++ b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.connectors.TestResource; +import internal.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepository; import ec.tss.tsproviders.cursor.TsCursor; import ec.tstoolkit.timeseries.simplets.TsData; diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java index 3b22cd6a..a02cea97 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/dotstat/DotStatPanel.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.web.SdmxWebManager; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import ec.nbdemetra.ui.completion.JAutoCompletionService; import ec.nbdemetra.ui.nodes.AbstractNodeBuilder; import ec.nbdemetra.ui.properties.NodePropertySetBuilder; @@ -247,7 +247,7 @@ boolean valid() { private static final class ConfigNode extends AbstractNode implements Editable { - public ConfigNode(WebEntryPoint o) { + public ConfigNode(SdmxWebEntryPoint o) { super(Children.LEAF, Lookups.singleton(o)); setDisplayName(o.getName()); setShortDescription(o.getDescription()); @@ -265,7 +265,7 @@ public Image getOpenedIcon(int type) { @Override protected Sheet createSheet() { - WebEntryPoint bean = getLookup().lookup(WebEntryPoint.class); + SdmxWebEntryPoint bean = getLookup().lookup(SdmxWebEntryPoint.class); Sheet result = new Sheet(); NodePropertySetBuilder b = new NodePropertySetBuilder(); b.with(String.class) @@ -297,7 +297,7 @@ public void actionPerformed(ActionEvent e) { @Override public void edit() { if (new PropertySheetDialogBuilder().title("Edit entry point").icon(getIcon(BeanInfo.ICON_MONO_16x16)).editNode(this)) { - setDisplayName(getLookup().lookup(WebEntryPoint.class).getName()); + setDisplayName(getLookup().lookup(SdmxWebEntryPoint.class).getName()); } } } diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java index 4bc84da3..a5dc5cfa 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java @@ -22,8 +22,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.web.SdmxWebManager; -import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import ec.nbdemetra.db.DbIcon; @@ -70,10 +69,7 @@ public static SdmxWebManager getDefaultManager() { private static SdmxWebManager createManager() { SdmxWebManager result = SdmxWebManager.of(Lookup.getDefault().lookupAll(SdmxWebDriver.class)); - ConcurrentMap cache = GuavaCaches.softValuesCacheAsMap(); - result.getDrivers().stream() - .filter(HasCache.class::isInstance) - .forEach(o -> ((HasCache) o).setCache(cache)); + result.setCache(GuavaCaches.softValuesCacheAsMap()); return result; } diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java index abb4283a..954fe187 100644 --- a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.web.SdmxWebManager; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.UnexpectedIOException; import com.google.common.base.Strings; import ec.util.completion.AutoCompletionSource; @@ -54,12 +54,12 @@ public AutoCompletionSource onEntryPoints(SdmxWebManager manager) { .builder(o -> manager.getEntryPoints()) .behavior(AutoCompletionSource.Behavior.SYNC) .postProcessor(SdmxAutoCompletion::filterAndSortEntryPoints) - .valueToString(WebEntryPoint::getName) + .valueToString(SdmxWebEntryPoint::getName) .build(); } public ListCellRenderer getEntryPointsRenderer() { - return CustomListCellRenderer.of(WebEntryPoint::getDescription, WebEntryPoint::getName); + return CustomListCellRenderer.of(SdmxWebEntryPoint::getDescription, SdmxWebEntryPoint::getName); } public AutoCompletionSource onFlows(SdmxConnectionSupplier supplier, LanguagePriorityList languages, Supplier source, ConcurrentMap cache) { @@ -103,11 +103,11 @@ public String getDefaultDimensionsAsString(SdmxConnectionSupplier supplier, Lang .collect(Collectors.joining(delimiter)); } - private List filterAndSortEntryPoints(List allValues, String term) { + private List filterAndSortEntryPoints(List allValues, String term) { Predicate filter = ExtAutoCompletionSource.basicFilter(term); return allValues.stream() .filter(o -> filter.test(o.getDescription()) || filter.test(o.getUri().toString())) - .sorted(Comparator.comparing(WebEntryPoint::getDescription)) + .sorted(Comparator.comparing(SdmxWebEntryPoint::getDescription)) .collect(Collectors.toList()); } diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index aa9f29b8..25c5a2dc 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -10,9 +10,9 @@ sdmx-facade-api sdmx-facade-util - sdmx-facade-connectors sdmx-facade-file sdmx-facade-samples + sdmx-facade-web @@ -63,7 +63,7 @@ ${project.groupId} - sdmx-facade-connectors + sdmx-facade-web ${project.version} diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java deleted file mode 100644 index 6b52d06f..00000000 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.web; - -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.ServiceLoader; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArrayList; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -public final class SdmxWebManager implements SdmxConnectionSupplier { - - @Nonnull - public static SdmxWebManager ofServiceLoader() { - return of(ServiceLoader.load(SdmxWebDriver.class)); - } - - @Nonnull - public static SdmxWebManager of(@Nonnull SdmxWebDriver... drivers) { - return of(Arrays.asList(drivers)); - } - - @Nonnull - public static SdmxWebManager of(@Nonnull Iterable drivers) { - CopyOnWriteArrayList driverList = new CopyOnWriteArrayList<>(); - ConcurrentMap entryPointByName = new ConcurrentHashMap<>(); - drivers.forEach(o -> { - driverList.add(o); - o.getDefaultEntryPoints().forEach(x -> entryPointByName.put(x.getName(), x)); - }); - return new SdmxWebManager(driverList, entryPointByName); - } - - private final CopyOnWriteArrayList drivers; - private final ConcurrentMap entryPointByName; - - private SdmxWebManager(CopyOnWriteArrayList drivers, ConcurrentMap entryPointByName) { - this.drivers = drivers; - this.entryPointByName = entryPointByName; - } - - @Override - public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { - WebEntryPoint wsEntryPoint = entryPointByName.get(name); - if (wsEntryPoint == null) { - throw new IOException("Cannot find entry point for '" + name + "'"); - } - URI uri = wsEntryPoint.getUri(); - try { - for (SdmxWebDriver o : drivers) { - if (o.acceptsURI(uri)) { - return o.connect(uri, wsEntryPoint.getProperties(), languages); - } - } - } catch (RuntimeException ex) { - throw new IOException("Failed to connect to '" + name + "'", ex); - } - throw new IOException("Failed to find a suitable driver for '" + name + "'"); - } - - @Nonnull - public List getDrivers() { - return Collections.unmodifiableList(drivers); - } - - public void setDrivers(@Nonnull List list) { - drivers.clear(); - drivers.addAll(list); - } - - @Nonnull - public List getEntryPoints() { - return new ArrayList<>(entryPointByName.values()); - } - - public void setEntryPoints(@Nonnull List list) { - entryPointByName.clear(); - list.forEach(o -> entryPointByName.put(o.getName(), o)); - } -} diff --git a/sdmx-facade/sdmx-facade-connectors/pom.xml b/sdmx-facade/sdmx-facade-web/pom.xml similarity index 94% rename from sdmx-facade/sdmx-facade-connectors/pom.xml rename to sdmx-facade/sdmx-facade-web/pom.xml index 2f7ea596..d0319c1b 100644 --- a/sdmx-facade/sdmx-facade-connectors/pom.xml +++ b/sdmx-facade/sdmx-facade-web/pom.xml @@ -8,7 +8,7 @@ 2.2.1-SNAPSHOT - sdmx-facade-connectors + sdmx-facade-web jar @@ -90,4 +90,5 @@ test + sdmx-facade-web \ No newline at end of file diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java similarity index 93% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java rename to sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java index 96264eb9..a888ff2b 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/WebEntryPoint.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java @@ -26,7 +26,7 @@ */ @lombok.Value @lombok.Builder(builderClassName = "Builder") -public final class WebEntryPoint { +public final class SdmxWebEntryPoint { @Nonnull String name; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java new file mode 100644 index 00000000..85b60241 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -0,0 +1,167 @@ +/* + * Copyright 2015 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.web; + +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.util.UnexpectedIOException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.AccessLevel; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +@lombok.extern.java.Log +public final class SdmxWebManager implements SdmxConnectionSupplier, HasCache { + + @Nonnull + public static SdmxWebManager ofServiceLoader() { + return of(ServiceLoader.load(SdmxWebDriver.class)); + } + + @Nonnull + public static SdmxWebManager of(@Nonnull SdmxWebDriver... drivers) { + return of(Arrays.asList(drivers)); + } + + @Nonnull + public static SdmxWebManager of(@Nonnull Iterable drivers) { + List driverList = new ArrayList<>(); + drivers.forEach(driverList::add); + + ConcurrentMap entryPointByName = new ConcurrentHashMap<>(); + ConcurrentMap cache = new ConcurrentHashMap(); + + updateEntryPointMap(entryPointByName, driverList.stream().flatMap(o -> tryGetDefaultEntryPoints(o).stream())); + updateCache(driverList, cache); + + return new SdmxWebManager(driverList, entryPointByName, new AtomicReference<>(cache)); + } + + private final List drivers; + private final ConcurrentMap entryPointByName; + private final AtomicReference cache; + + @Override + public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + SdmxWebEntryPoint entryPoint = entryPointByName.get(name); + if (entryPoint == null) { + throw new IOException("Cannot find entry point for '" + name + "'"); + } + for (SdmxWebDriver o : drivers) { + if (tryAcceptURI(o, entryPoint)) { + return tryConnect(o, entryPoint, languages); + } + } + throw new IOException("Failed to find a suitable driver for '" + name + "'"); + } + + @Override + public ConcurrentMap getCache() { + return cache.get(); + } + + @Override + public void setCache(ConcurrentMap cache) { + this.cache.set(cache != null ? cache : new ConcurrentHashMap()); + updateCache(drivers, this.cache.get()); + } + + @Nonnull + public List getEntryPoints() { + return new ArrayList<>(entryPointByName.values()); + } + + public void setEntryPoints(@Nonnull List list) { + updateEntryPointMap(entryPointByName, list.stream()); + } + + private static void updateCache(List drivers, ConcurrentMap cache) { + drivers.stream() + .filter(HasCache.class::isInstance) + .forEach(o -> ((HasCache) o).setCache(cache)); + } + + private static void updateEntryPointMap(ConcurrentMap entryPointByName, Stream list) { + entryPointByName.clear(); + list.forEach(o -> entryPointByName.put(o.getName(), o)); + } + + private static boolean tryAcceptURI(SdmxWebDriver driver, SdmxWebEntryPoint entryPoint) throws IOException { + try { + return driver.acceptsURI(entryPoint.getUri()); + } catch (RuntimeException ex) { + log.log(Level.WARNING, "Unexpected exception while parsing URI", ex); + return false; + } + } + + @SuppressWarnings("null") + private static SdmxConnection tryConnect(SdmxWebDriver driver, SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + SdmxConnection result; + + try { + result = driver.connect(entryPoint.getUri(), entryPoint.getProperties(), languages); + } catch (RuntimeException ex) { + log.log(Level.WARNING, "Unexpected exception while connecting", ex); + throw new UnexpectedIOException(ex); + } + + if (result == null) { + log.log(Level.WARNING, "Unexpected null connection"); + throw new IOException("Unexpected null connection"); + } + + return result; + } + + @SuppressWarnings("null") + private static Collection tryGetDefaultEntryPoints(SdmxWebDriver driver) { + Collection result; + + try { + result = driver.getDefaultEntryPoints(); + } catch (RuntimeException ex) { + log.log(Level.WARNING, "Unexpected exception while getting default entry points", ex); + return Collections.emptyList(); + } + + if (result == null) { + log.log(Level.WARNING, "Unexpected null list"); + return Collections.emptyList(); + } + + return result; + } +} diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java similarity index 89% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java index b9aa656d..7fbc0e7f 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/web/SdmxWebDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java @@ -14,10 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.web; +package be.nbb.sdmx.facade.web.spi; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import java.io.IOException; import java.net.URI; import java.util.Collection; @@ -38,5 +39,5 @@ public interface SdmxWebDriver { boolean acceptsURI(@Nonnull URI uri) throws IOException; @Nonnull - Collection getDefaultEntryPoints(); + Collection getDefaultEntryPoints(); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java index 8c412ef3..d95d30fa 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ABS; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class AbsDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ABS.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java similarity index 99% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java index 19e29c79..bc8fe914 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/CachedSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ClientSupplier.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java similarity index 97% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ClientSupplier.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java index c3925f5a..adec81bf 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ClientSupplier.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.LanguagePriorityList; import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/DataCursorAdapter.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java similarity index 95% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/DataCursorAdapter.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java index 8362772b..c3428320 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/DataCursorAdapter.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java index 5ce10440..5baf5c04 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.EUROSTAT; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class EurostatDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, EUROSTAT.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasDataCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java similarity index 96% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasDataCursor.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java index 022e114d..75875067 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasDataCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasSeriesKeysOnlySupported.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java similarity index 95% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasSeriesKeysOnlySupported.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java index 438acec6..a0d4fd29 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/HasSeriesKeysOnlySupported.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; /** * diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java index f8ed1237..bb7b5195 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/IloDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ILO; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class IloDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ILO.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java index 86ca1e5f..9fd59b8a 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.IMF; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class ImfDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, IMF.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java similarity index 97% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java index 3c7a3c15..b1cd2e78 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -26,7 +26,7 @@ import static be.nbb.sdmx.facade.Frequency.UNDEFINED; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.FreqParser; import be.nbb.sdmx.facade.util.HasCache; @@ -52,7 +52,7 @@ import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -69,7 +69,7 @@ public final class InseeDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l, xml)); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java index e21ef6fb..886d8fab 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.NBB; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class NbbDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, NBB.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java index eb238e38..099b1286 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.OECD; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class OecdDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, OECD.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java index 1a134941..0a3e3290 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java @@ -14,11 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.LanguagePriorityList; -import static be.nbb.sdmx.facade.connectors.Util.NEEDS_CREDENTIALS; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import static internal.connectors.Util.NEEDS_CREDENTIALS; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.RestSdmx20Client; import java.net.URL; @@ -26,7 +26,7 @@ import java.util.List; import java.util.Map; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -41,7 +41,7 @@ public final class Sdmx20Driver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, Sdmx20Client::new); @Override - public List getDefaultEntryPoints() { + public List getDefaultEntryPoints() { return Collections.emptyList(); } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Config.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java similarity index 97% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Config.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java index 815db1c2..55b889ae 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Config.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import java.util.Map; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java similarity index 95% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java index 8f713f24..c3e5e561 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java @@ -14,13 +14,13 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.SdmxMediaType; @@ -40,7 +40,7 @@ import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -57,9 +57,9 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l, xml)); @Override - public List getDefaultEntryPoints() { + public List getDefaultEntryPoints() { Sdmx21EntryPointBuilder b = new Sdmx21EntryPointBuilder(); - List result = new ArrayList<>(); + List result = new ArrayList<>(); result.add(b.clear() .name("ECB") .description("European Central Bank") @@ -157,8 +157,8 @@ private Map toProperties() { return result; } - public WebEntryPoint build() { - return WebEntryPoint.builder() + public SdmxWebEntryPoint build() { + return SdmxWebEntryPoint.builder() .name(name) .description(description) .uri(PREFIX + endpoint) diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java similarity index 99% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java index 2c40a0b0..7f037940 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapter.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java similarity index 92% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java index e11b24bb..72171669 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/SdmxDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java @@ -14,11 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CACHE_TTL; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CONNECT_TIMEOUT; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.READ_TIMEOUT; @@ -95,8 +95,8 @@ public void setCache(ConcurrentMap cache) { } @Nonnull - public static Collection entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { - return Collections.singleton(WebEntryPoint.builder().name(name).description(description).uri(url).build()); + public static Collection entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { + return Collections.singleton(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); } // diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java index 4ac82a96..adb3b868 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/UisDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.UIS; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class UisDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, UIS.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java similarity index 98% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Util.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 9fd56034..65b23cb4 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java similarity index 87% rename from sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java index 1e6f16e5..b428f141 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/main/java/be/nbb/sdmx/facade/connectors/WbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; -import be.nbb.sdmx.facade.web.WebEntryPoint; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.WB; import java.util.Collection; import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.SdmxWebDriver; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; /** * @@ -36,7 +36,7 @@ public final class WbDriver implements SdmxWebDriver, HasCache { private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, WB.class); @Override - public Collection getDefaultEntryPoints() { + public Collection getDefaultEntryPoints() { return SdmxDriverSupport.entry("WB", "World Bank", "sdmx:wb:http://api.worldbank.org"); } } diff --git a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java similarity index 97% rename from sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ConnectorsResource.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index e487e74b..e3f4e391 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; diff --git a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/DataCursorAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java similarity index 98% rename from sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/DataCursorAdapterTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java index 1b385a3d..1f8a1119 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/DataCursorAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; diff --git a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java similarity index 99% rename from sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/FacadeResource.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index 0b63d7e5..7c4fe463 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; diff --git a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ParsersTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ParsersTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java index 02555a2a..c77a8646 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/ParsersTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; diff --git a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapterTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java index e8e8ac43..c201a019 100644 --- a/sdmx-facade/sdmx-facade-connectors/src/test/java/be/nbb/sdmx/facade/connectors/SdmxConnectionAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.connectors; +package internal.connectors; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.tck.ConnectionAssert; From 0c002464df738275d23f60e3b367606c90c61f06 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 17 Oct 2017 16:31:58 +0200 Subject: [PATCH 08/68] Refactored sdmx file manager. --- .../demetra/sdmx/file/SdmxFileProvider.java | 4 ++-- .../nbb/sdmx/facade/file/SdmxFileManager.java | 18 +++++++++++------- .../file/CachedFileSdmxConnection.java | 7 ++++--- .../file}/CustomDataStructureBuilder.java | 4 ++-- .../file}/DataStructureDecoder.java | 11 +++++------ .../impl => internal/file}/DataTypeProbe.java | 13 ++++++------- .../file/FileSdmxConnection.java | 3 ++- .../facade => internal}/file/SdmxDecoder.java | 3 ++- .../file}/XMLStreamSdmxDecoder.java | 11 +++++------ .../file}/DataStructureDecoderTest.java | 5 ++--- .../file}/DataTypeProbeTest.java | 3 +-- .../file/FileSdmxConnectionTest.java | 4 ++-- 12 files changed, 44 insertions(+), 42 deletions(-) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade => internal}/file/CachedFileSdmxConnection.java (91%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade/file/impl => internal/file}/CustomDataStructureBuilder.java (98%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade/file/impl => internal/file}/DataStructureDecoder.java (97%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade/file/impl => internal/file}/DataTypeProbe.java (90%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade => internal}/file/FileSdmxConnection.java (98%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade => internal}/file/SdmxDecoder.java (95%) rename sdmx-facade/sdmx-facade-file/src/main/java/{be/nbb/sdmx/facade/file/impl => internal/file}/XMLStreamSdmxDecoder.java (89%) rename sdmx-facade/sdmx-facade-file/src/test/java/{be/nbb/sdmx/facade/file/impl => internal/file}/DataStructureDecoderTest.java (96%) rename sdmx-facade/sdmx-facade-file/src/test/java/{be/nbb/sdmx/facade/file/impl => internal/file}/DataTypeProbeTest.java (96%) rename sdmx-facade/sdmx-facade-file/src/test/java/{be/nbb/sdmx/facade => internal}/file/FileSdmxConnectionTest.java (97%) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 4b08124b..b77a896f 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -80,7 +80,7 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private final ITsProvider tsSupport; public SdmxFileProvider() { - this.connectionSupplier = new AtomicReference<>(new SdmxFileManager()); + this.connectionSupplier = new AtomicReference<>(SdmxFileManager.of()); this.languages = new AtomicReference<>(LanguagePriorityList.ANY); Logger logger = LoggerFactory.getLogger(NAME); @@ -113,7 +113,7 @@ public SdmxConnectionSupplier getConnectionSupplier() { @Override public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : new SdmxFileManager())) { + if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxFileManager.of())) { clearCache(); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 85e02c2d..35171e8a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -16,10 +16,12 @@ */ package be.nbb.sdmx.facade.file; +import internal.file.CachedFileSdmxConnection; +import internal.file.SdmxDecoder; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.file.impl.XMLStreamSdmxDecoder; +import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; @@ -27,23 +29,25 @@ import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.xml.stream.XMLInputFactory; +import lombok.AccessLevel; /** * * @author Philippe Charles */ +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) public final class SdmxFileManager implements SdmxConnectionSupplier, HasCache { + @Nonnull + public static SdmxFileManager of() { + XMLInputFactory factory = XMLInputFactory.newInstance(); + return new SdmxFileManager(factory, new XMLStreamSdmxDecoder(factory), new AtomicReference<>(new ConcurrentHashMap())); + } + private final XMLInputFactory factory; private final SdmxDecoder decoder; private final AtomicReference cache; - public SdmxFileManager() { - this.factory = XMLInputFactory.newInstance(); - this.decoder = new XMLStreamSdmxDecoder(factory); - this.cache = new AtomicReference<>(new ConcurrentHashMap()); - } - @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { try { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java similarity index 91% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/CachedFileSdmxConnection.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 5e68378a..50341b48 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -14,12 +14,13 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file; +package internal.file; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; @@ -35,7 +36,7 @@ * * @author Philippe Charles */ -final class CachedFileSdmxConnection extends FileSdmxConnection { +public final class CachedFileSdmxConnection extends FileSdmxConnection { // TODO: replace ttl with file last modification time private static final long DEFAULT_CACHE_TTL = TimeUnit.MINUTES.toMillis(5); @@ -45,7 +46,7 @@ final class CachedFileSdmxConnection extends FileSdmxConnection { private final TypedId decodeKey; private final TypedId> loadDataKey; - CachedFileSdmxConnection(SdmxFile file, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { + public CachedFileSdmxConnection(SdmxFile file, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { super(file, languages, factory, decoder); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); String base = file.toString() + languages.toString(); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/CustomDataStructureBuilder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java similarity index 98% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/CustomDataStructureBuilder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java index 3534cb3e..1c72d685 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/CustomDataStructureBuilder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; @@ -29,7 +29,7 @@ import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import be.nbb.sdmx.facade.file.SdmxDecoder.DataType; +import internal.file.SdmxDecoder.DataType; import java.util.Collection; /** diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java similarity index 97% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index e980c7ba..d7cbe7bc 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -14,14 +14,13 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.file.SdmxDecoder; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT21; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC21; +import static internal.file.SdmxDecoder.DataType.COMPACT20; +import static internal.file.SdmxDecoder.DataType.COMPACT21; +import static internal.file.SdmxDecoder.DataType.GENERIC20; +import static internal.file.SdmxDecoder.DataType.GENERIC21; import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; import java.io.IOException; import java.io.Reader; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java similarity index 90% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataTypeProbe.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java index 1e723d7f..76e62c27 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java @@ -14,14 +14,13 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; -import be.nbb.sdmx.facade.file.SdmxDecoder; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT21; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC21; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.UNKNOWN; +import static internal.file.SdmxDecoder.DataType.COMPACT20; +import static internal.file.SdmxDecoder.DataType.COMPACT21; +import static internal.file.SdmxDecoder.DataType.GENERIC20; +import static internal.file.SdmxDecoder.DataType.GENERIC21; +import static internal.file.SdmxDecoder.DataType.UNKNOWN; import java.io.IOException; import java.io.Reader; import javax.xml.stream.XMLInputFactory; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java similarity index 98% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index c108d079..ae768ad6 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file; +package internal.file; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -26,6 +26,7 @@ import be.nbb.sdmx.facade.QueryDetail; import be.nbb.sdmx.facade.QueryParameters; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.io.IOException; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java similarity index 95% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxDecoder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java index d60ce238..c7f99794 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java @@ -14,10 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file; +package internal.file; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.file.SdmxFile; import java.io.IOException; import javax.annotation.Nonnull; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java similarity index 89% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/XMLStreamSdmxDecoder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index c10ddfd5..a4ded188 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/impl/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.LanguagePriorityList; @@ -22,11 +22,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import javax.xml.stream.XMLInputFactory; -import be.nbb.sdmx.facade.file.SdmxDecoder; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.COMPACT21; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC20; -import static be.nbb.sdmx.facade.file.SdmxDecoder.DataType.GENERIC21; +import static internal.file.SdmxDecoder.DataType.COMPACT20; +import static internal.file.SdmxDecoder.DataType.COMPACT21; +import static internal.file.SdmxDecoder.DataType.GENERIC20; +import static internal.file.SdmxDecoder.DataType.GENERIC21; import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoderTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index f9b64874..dd156583 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -14,17 +14,16 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import java.io.IOException; import javax.xml.stream.XMLInputFactory; import org.junit.Test; -import be.nbb.sdmx.facade.file.SdmxDecoder; import be.nbb.sdmx.facade.samples.SdmxSource; import static org.assertj.core.api.Assertions.assertThat; -import static be.nbb.sdmx.facade.file.impl.CustomDataStructureBuilder.dimension; +import static internal.file.CustomDataStructureBuilder.dimension; import java.io.Reader; /** diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataTypeProbeTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java index 311b3d7e..e14033a7 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/impl/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java @@ -14,9 +14,8 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file.impl; +package internal.file; -import be.nbb.sdmx.facade.file.SdmxDecoder; import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import org.junit.Test; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 1eb60516..2906f473 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -14,14 +14,14 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.file; +package internal.file; -import be.nbb.sdmx.facade.file.impl.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; import java.io.File; From fb7915af25ccab2715ab6f2b2c576840d838d44f Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 17 Oct 2017 17:23:34 +0200 Subject: [PATCH 09/68] Improved tests. --- .../sdmx/file/SdmxFileProviderTest.java | 2 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 6 +- .../sdmx/facade/file/SdmxFileManagerTest.java | 45 ++++++++++ .../internal/file/FileSdmxConnectionTest.java | 2 +- .../nbb/sdmx/facade/samples/ByteSource.java | 5 ++ .../facade/tck/ConnectionSupplierAssert.java | 61 +++++++++++++ .../nbb/sdmx/facade/repo/SdmxRepository.java | 12 +-- .../facade/repo/SdmxRepositoryManager.java | 4 + .../repo/SdmxRepositoryManagerTest.java | 33 +++++++ .../sdmx/facade/web/SdmxWebEntryPoint.java | 11 +-- .../nbb/sdmx/facade/web/SdmxWebManager.java | 14 ++- .../sdmx/facade/web/SdmxWebManagerTest.java | 89 +++++++++++++++++++ 12 files changed, 266 insertions(+), 18 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java create mode 100644 sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManagerTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java index 842f22c5..7d6c4a49 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java @@ -48,7 +48,7 @@ public class SdmxFileProviderTest { private static File createTemp(ByteSource bytes, String prefix, String suffix) throws IOException { File result = File.createTempFile(prefix, suffix); result.deleteOnExit(); - bytes.copyTo(result.toPath()); + bytes.copyTo(result); return result; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 35171e8a..5a3ec94a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -50,11 +50,15 @@ public static SdmxFileManager of() { @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + SdmxFile file; + try { - return getConnection(SdmxFile.parse(name), languages); + file = SdmxFile.parse(name); } catch (IllegalArgumentException ex) { throw new IOException(ex.getMessage(), ex.getCause()); } + + return getConnection(file, languages); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java new file mode 100644 index 00000000..4d7b245a --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.file; + +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; +import java.io.File; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * + * @author Philippe Charles + */ +public class SdmxFileManagerTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void testCompliance() throws IOException { + File compact21 = temp.newFile(); + SdmxSource.OTHER_COMPACT21.copyTo(compact21); + + SdmxFile file = new SdmxFile(compact21, null); + + ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), file.toString(), "ko"); + } +} diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 2906f473..d8d8326a 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -46,7 +46,7 @@ public class FileSdmxConnectionTest { @Test public void testCompactData21() throws IOException { File compact21 = temp.newFile(); - SdmxSource.OTHER_COMPACT21.copyTo(compact21.toPath()); + SdmxSource.OTHER_COMPACT21.copyTo(compact21); SdmxFile file = new SdmxFile(compact21, null); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java index de2b01f7..816987d2 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.samples; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -54,6 +55,10 @@ default void copyTo(@Nonnull Path file) throws IOException { } } + default void copyTo(@Nonnull File file) throws IOException { + copyTo(file.toPath()); + } + @Nonnull static ByteSource of(@Nonnull Class type, @Nonnull String name) { Objects.requireNonNull(type); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java new file mode 100644 index 00000000..f2d84b53 --- /dev/null +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java @@ -0,0 +1,61 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.tck; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import java.io.IOException; +import org.assertj.core.api.SoftAssertions; + +/** + * + * @author Philippe Charles + */ +public final class ConnectionSupplierAssert { + + public static void assertCompliance(SdmxConnectionSupplier supplier, String validName, String invalidName) { + SoftAssertions s = new SoftAssertions(); + try { + assertCompliance(s, supplier, validName, invalidName); + } catch (Exception ex) { + s.fail("Unexpected exception '" + ex.getClass() + "'", ex); + } + s.assertAll(); + } + + public static void assertCompliance(SoftAssertions s, SdmxConnectionSupplier supplier, String name, String invalidName) throws Exception { + assertNonnull(s, supplier, name); + + s.assertThatThrownBy(() -> supplier.getConnection(invalidName, LanguagePriorityList.ANY)).isInstanceOf(IOException.class); + + try (SdmxConnection conn = supplier.getConnection(name, LanguagePriorityList.ANY)) { + s.assertThat(conn).isNotNull(); + } + } + + @SuppressWarnings("null") + private static void assertNonnull(SoftAssertions s, SdmxConnectionSupplier supplier, String name) { + s.assertThatThrownBy(() -> supplier.getConnection(null, LanguagePriorityList.ANY)) + .as("Expecting 'getConnection(String, LanguagePriorityList)' to raise NPE when called with null name") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> supplier.getConnection(name, null)) + .as("Expecting 'getConnection(String, LanguagePriorityList)' to raise NPE when called with null language") + .isInstanceOf(NullPointerException.class); + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 4e74b515..16194e5b 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -72,19 +72,16 @@ public SdmxConnection asConnection() { public static final class Builder { + private final Map> data = new HashMap<>(); + @Nonnull public Builder clearData() { - if (this.data == null) { - this.data.clear(); - } + this.data.clear(); return this; } @Nonnull public Builder data(@Nonnull DataflowRef flowRef, @Nonnull Series series) { - if (this.data == null) { - this.data = new HashMap<>(); - } data.computeIfAbsent(flowRef, o -> new ArrayList<>()).add(series); return this; } @@ -92,9 +89,6 @@ public Builder data(@Nonnull DataflowRef flowRef, @Nonnull Series series) { @Nonnull public Builder data(@Nonnull DataflowRef flowRef, @Nonnull List list) { if (!list.isEmpty()) { - if (this.data == null) { - this.data = new HashMap<>(); - } data.computeIfAbsent(flowRef, o -> new ArrayList<>()).addAll(list); } return this; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java index b522e407..08a2855e 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManager.java @@ -21,6 +21,7 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import java.io.IOException; import java.util.List; +import java.util.Objects; /** * @@ -36,6 +37,9 @@ public final class SdmxRepositoryManager implements SdmxConnectionSupplier { @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + Objects.requireNonNull(name); + Objects.requireNonNull(languages); + return repositories.stream() .filter(o -> o.getName().equals(name)) .findFirst() diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManagerTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManagerTest.java new file mode 100644 index 00000000..47c5adff --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryManagerTest.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.repo; + +import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxRepositoryManagerTest { + + @Test + public void testCompliance() { + SdmxRepository repo = SdmxRepository.builder().name("ok").build(); + ConnectionSupplierAssert.assertCompliance(SdmxRepositoryManager.builder().repository(repo).build(), "ok", "ko"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java index a888ff2b..1157cbdd 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebEntryPoint.java @@ -25,16 +25,17 @@ * @author Philippe Charles */ @lombok.Value -@lombok.Builder(builderClassName = "Builder") +@lombok.Builder(builderClassName = "Builder", toBuilder = true) public final class SdmxWebEntryPoint { - @Nonnull + @lombok.NonNull String name; - @Nonnull - String description; + @lombok.NonNull + @lombok.Builder.Default + String description = ""; - @Nonnull + @lombok.NonNull URI uri; @lombok.Singular diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 85b60241..3664523b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -75,16 +76,27 @@ public static SdmxWebManager of(@Nonnull Iterable drive @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + Objects.requireNonNull(name); + Objects.requireNonNull(languages); + SdmxWebEntryPoint entryPoint = entryPointByName.get(name); if (entryPoint == null) { throw new IOException("Cannot find entry point for '" + name + "'"); } + return getConnection(entryPoint, languages); + } + + @Nonnull + public SdmxConnection getConnection(@Nonnull SdmxWebEntryPoint entryPoint, @Nonnull LanguagePriorityList languages) throws IOException { + Objects.requireNonNull(entryPoint); + Objects.requireNonNull(languages); + for (SdmxWebDriver o : drivers) { if (tryAcceptURI(o, entryPoint)) { return tryConnect(o, entryPoint, languages); } } - throw new IOException("Failed to find a suitable driver for '" + name + "'"); + throw new IOException("Failed to find a suitable driver for '" + entryPoint + "'"); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java new file mode 100644 index 00000000..00726132 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.web; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxWebManagerTest { + + @Test + public void testCompliance() { + ConnectionSupplierAssert.assertCompliance(SdmxWebManager.of(REPO), HELLO.getName(), "ko"); + } + + @Test + @SuppressWarnings("null") + public void testFactories() { + assertThatThrownBy(() -> SdmxWebManager.of((Iterable) null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxWebManager.of((SdmxWebDriver[]) null)).isInstanceOf(NullPointerException.class); + } + + @Test + @SuppressWarnings("null") + public void testGetConnectionOfEntryPoint() { + SdmxWebManager manager = SdmxWebManager.of(REPO); + assertThatThrownBy(() -> manager.getConnection((SdmxWebEntryPoint) null, LanguagePriorityList.ANY)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> manager.getConnection(HELLO, null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> manager.getConnection(HELLO.toBuilder().uri("ko").build(), LanguagePriorityList.ANY)).isInstanceOf(IOException.class); + } + + private static final SdmxWebEntryPoint HELLO = SdmxWebEntryPoint.builder().name("ok").uri(RepoDriver.PREFIX + "r1").build(); + private static final SdmxWebDriver REPO = new RepoDriver(); + + private static final class RepoDriver implements SdmxWebDriver { + + static final String PREFIX = "sdmx:repo:"; + + final List repos = Collections.singletonList(SdmxRepository.builder().name("r1").build()); + + @Override + public SdmxConnection connect(URI uri, Map properties, LanguagePriorityList languages) throws IOException { + String repoName = uri.toString().substring(PREFIX.length()); + return repos.stream() + .filter(o -> o.getName().equals(repoName)) + .findFirst() + .map(SdmxRepository::asConnection) + .orElseThrow(IOException::new); + } + + @Override + public boolean acceptsURI(URI uri) throws IOException { + return uri.toString().startsWith(PREFIX); + } + + @Override + public Collection getDefaultEntryPoints() { + return Collections.singletonList(HELLO); + } + } +} From 1114d7cc4e519e14ebd91117254343a4ba3fa93f Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 18 Oct 2017 11:28:15 +0200 Subject: [PATCH 10/68] Added SdmxConnectionSupplier#getConnection(String). --- .../java/be/nbb/demetra/dotstat/DotStatAccessorTest.java | 4 ++-- .../java/be/nbb/sdmx/facade/SdmxConnectionSupplier.java | 5 +++++ .../be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index a660fcbd..00ff1822 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -80,7 +80,7 @@ private static DbSetId ecbRoot() { @Test public void testGetKey() throws Exception { - DataStructure dfs = supplier.getConnection("NBB", LanguagePriorityList.ANY).getDataStructure(DataflowRef.of("NBB", "TEST_DATASET", null)); + DataStructure dfs = supplier.getConnection("NBB").getDataStructure(DataflowRef.of("NBB", "TEST_DATASET", null)); Map dimById = DotStatAccessor.dimensionById(dfs); // default ordering of dimensions @@ -100,7 +100,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { - try (DataCursor o = supplier.getConnection("NBB", LanguagePriorityList.ANY).getData(DataflowRef.of("NBB", "TEST_DATASET", null), Key.ALL, QueryParameters.SERIES_KEYS_ONLY)) { + try (DataCursor o = supplier.getConnection("NBB").getData(DataflowRef.of("NBB", "TEST_DATASET", null), Key.ALL, QueryParameters.SERIES_KEYS_ONLY)) { o.nextSeries(); assertThat(o.getSeriesKey()).isEqualTo(Key.parse("LOCSTL04.AUS.M")); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnectionSupplier.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnectionSupplier.java index cd7239f4..fdd78cb3 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnectionSupplier.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnectionSupplier.java @@ -29,4 +29,9 @@ public interface SdmxConnectionSupplier { @Nonnull SdmxConnection getConnection(@Nonnull String name, @Nonnull LanguagePriorityList languages) throws IOException; + + @Nonnull + default SdmxConnection getConnection(@Nonnull String name) throws IOException { + return getConnection(name, LanguagePriorityList.ANY); + } } diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java index f2d84b53..40826215 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionSupplierAssert.java @@ -41,15 +41,19 @@ public static void assertCompliance(SdmxConnectionSupplier supplier, String vali public static void assertCompliance(SoftAssertions s, SdmxConnectionSupplier supplier, String name, String invalidName) throws Exception { assertNonnull(s, supplier, name); - s.assertThatThrownBy(() -> supplier.getConnection(invalidName, LanguagePriorityList.ANY)).isInstanceOf(IOException.class); + s.assertThatThrownBy(() -> supplier.getConnection(invalidName)).isInstanceOf(IOException.class); - try (SdmxConnection conn = supplier.getConnection(name, LanguagePriorityList.ANY)) { + try (SdmxConnection conn = supplier.getConnection(name)) { s.assertThat(conn).isNotNull(); } } @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnectionSupplier supplier, String name) { + s.assertThatThrownBy(() -> supplier.getConnection(null)) + .as("Expecting 'getConnection(String, LanguagePriorityList)' to raise NPE when called with null name") + .isInstanceOf(NullPointerException.class); + s.assertThatThrownBy(() -> supplier.getConnection(null, LanguagePriorityList.ANY)) .as("Expecting 'getConnection(String, LanguagePriorityList)' to raise NPE when called with null name") .isInstanceOf(NullPointerException.class); From c3a6e96a62204fa92812f309d6b0a199846005bc Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 18 Oct 2017 11:52:31 +0200 Subject: [PATCH 11/68] Merged Key + DataParameter into DataQuery. --- .../main/java/internal/sdmx/SdmxQueryUtil.java | 4 ++-- .../demetra/dotstat/DotStatAccessorTest.java | 4 ++-- .../{QueryParameters.java => DataQuery.java} | 18 ++++++++++++++---- .../{QueryDetail.java => DataQueryDetail.java} | 2 +- .../be/nbb/sdmx/facade/SdmxConnection.java | 2 +- .../java/internal/file/FileSdmxConnection.java | 10 +++++----- .../internal/file/FileSdmxConnectionTest.java | 4 ++-- .../nbb/sdmx/facade/tck/ConnectionAssert.java | 12 ++++++------ .../nbb/sdmx/facade/repo/SdmxRepository.java | 9 ++++----- .../connectors/CachedSdmxConnection.java | 4 ++-- .../connectors/SdmxConnectionAdapter.java | 10 +++++----- 11 files changed, 44 insertions(+), 35 deletions(-) rename sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/{QueryParameters.java => DataQuery.java} (67%) rename sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/{QueryDetail.java => DataQueryDetail.java} (96%) diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index 493bcda2..b1a598b5 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import com.google.common.collect.ImmutableList; import ec.tss.tsproviders.cursor.TsCursor; @@ -88,7 +88,7 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re private final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { - return new SdmxDataAdapter(key, conn.getData(flowRef, key, !seriesKeysOnly ? QueryParameters.FULL : QueryParameters.SERIES_KEYS_ONLY), labelAttribute); + return new SdmxDataAdapter(key, conn.getData(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); } private TsCursor computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index 00ff1822..3fc6771b 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import internal.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; @@ -100,7 +100,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { - try (DataCursor o = supplier.getConnection("NBB").getData(DataflowRef.of("NBB", "TEST_DATASET", null), Key.ALL, QueryParameters.SERIES_KEYS_ONLY)) { + try (DataCursor o = supplier.getConnection("NBB").getData(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true))) { o.nextSeries(); assertThat(o.getSeriesKey()).isEqualTo(Key.parse("LOCSTL04.AUS.M")); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java similarity index 67% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java index 056659de..456f170c 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryParameters.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java @@ -16,18 +16,28 @@ */ package be.nbb.sdmx.facade; +import javax.annotation.Nonnull; + /** * * @author Philippe Charles */ @lombok.Value @lombok.Builder(builderClassName = "Builder") -public class QueryParameters { +public class DataQuery { - public static final QueryParameters FULL = builder().detail(QueryDetail.FULL).build(); - public static final QueryParameters SERIES_KEYS_ONLY = builder().detail(QueryDetail.SERIES_KEYS_ONLY).build(); + @lombok.NonNull + Key key; @lombok.NonNull @lombok.Builder.Default - private QueryDetail detail = QueryDetail.FULL; + DataQueryDetail detail = DataQueryDetail.FULL; + + @Nonnull + public static DataQuery of(@Nonnull Key key, boolean seriesKeysOnly) { + return DataQuery.builder() + .key(key) + .detail(seriesKeysOnly ? DataQueryDetail.SERIES_KEYS_ONLY : DataQueryDetail.FULL) + .build(); + } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java similarity index 96% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java index ef14ce2d..a460c90e 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/QueryDetail.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java @@ -20,7 +20,7 @@ * * @author Philippe Charles */ -public enum QueryDetail { +public enum DataQueryDetail { FULL, SERIES_KEYS_ONLY } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java index 1eec2d14..ef4e858f 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java @@ -39,7 +39,7 @@ public interface SdmxConnection extends Closeable { DataStructure getDataStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull QueryParameters queryParams) throws IOException; + DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index ae768ad6..2d4644d0 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -23,8 +23,8 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.QueryDetail; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; @@ -83,12 +83,12 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { + final public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); Objects.requireNonNull(flowRef); - Objects.requireNonNull(key); + Objects.requireNonNull(query); checkFlowRef(flowRef); - return loadData(decode(), flowRef, key, queryParams.getDetail().equals(QueryDetail.SERIES_KEYS_ONLY)); + return loadData(decode(), flowRef, query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); } @Override diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index d8d8326a..e3e98be0 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.file.SdmxFile; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; @@ -57,7 +57,7 @@ public void testCompactData21() throws IOException { Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getData(file.getDataflowRef(), Key.ALL, QueryParameters.FULL)) { + try (DataCursor o = conn.getData(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 15c513af..a9d20ea4 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -19,7 +19,7 @@ import internal.io.ConsumerWithIO; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import java.io.IOException; import java.util.concurrent.Callable; @@ -52,7 +52,7 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getData(ref, Key.ALL, QueryParameters.FULL), "getData(DataFlowRef, Key, boolean)"); + assertState(s, supplier, o -> o.getData(ref, DataQuery.of(Key.ALL, false)), "getData(DataFlowRef, DataQuery)"); assertState(s, supplier, o -> o.getDataStructure(ref), "getDataStructure(DataFlowRef)"); assertState(s, supplier, o -> o.getDataflow(ref), "getDataflow(DataFlowRef)"); assertState(s, supplier, SdmxConnection::getDataflows, "getDataflows()"); @@ -60,12 +60,12 @@ public static void assertCompliance(SoftAssertions s, Callable s @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getData(null, Key.ALL, QueryParameters.FULL)) - .as("Expecting 'getData(DataFlowRef, Key, boolean)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getData(null, DataQuery.of(Key.ALL, false))) + .as("Expecting 'getData(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getData(ref, null, QueryParameters.FULL)) - .as("Expecting 'getData(DataFlowRef, Key, boolean)' to raise NPE when called with null key") + s.assertThatThrownBy(() -> conn.getData(ref, null)) + .as("Expecting 'getData(DataFlowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getDataStructure(null)) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 16194e5b..d576edc7 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -21,8 +21,7 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import java.io.IOException; import java.util.ArrayList; @@ -145,13 +144,13 @@ public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); Objects.requireNonNull(flowRef); - Objects.requireNonNull(key); + Objects.requireNonNull(query); List col = data.get(flowRef); if (col != null) { - return Series.asCursor(col, key); + return Series.asCursor(col, query.getKey()); } throw new IOException("Data not found"); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java index bc8fe914..30cff6ef 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; @@ -109,7 +109,7 @@ protected DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeyson } cache.put(id, result); } - return result.asConnection().getData(flowRef, key, QueryParameters.SERIES_KEYS_ONLY); + return result.asConnection().getData(flowRef, DataQuery.of(key, true)); } return super.loadData(flowRef, key, serieskeysonly); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java index 7f037940..1deb2c97 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java @@ -21,8 +21,8 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.QueryDetail; -import be.nbb.sdmx.facade.QueryParameters; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.util.NoOpCursor; import be.nbb.sdmx.facade.util.ObsParser; @@ -75,13 +75,13 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getData(DataflowRef flowRef, Key key, QueryParameters queryParams) throws IOException { + final public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - boolean serieskeysonly = queryParams.getDetail().equals(QueryDetail.SERIES_KEYS_ONLY); + boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); if (serieskeysonly && !isSeriesKeysOnlySupported()) { throw new IllegalStateException("serieskeysonly not supported"); } - return loadData(flowRef, key, serieskeysonly); + return loadData(flowRef, query.getKey(), serieskeysonly); } @Override From 106e15d15b282979d3e0bd203d3e6b266e958963 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 18 Oct 2017 14:23:41 +0200 Subject: [PATCH 12/68] Refactored connectors drivers. --- ...dmxConnection.java => CachedResource.java} | 53 +++--- .../connectors/ConnectorsConnection.java | 109 ++++++++++++ ...port.java => ConnectorsDriverSupport.java} | 36 ++-- .../connectors/GenericSDMXClientResource.java | 160 +++++++++++++++++ ...er.java => GenericSDMXClientSupplier.java} | 4 +- .../internal/connectors/HasDataCursor.java | 2 +- .../HasSeriesKeysOnlySupported.java | 2 +- .../java/internal/connectors/InseeDriver.java | 168 ------------------ ...ter.java => PortableTimeSeriesCursor.java} | 4 +- .../connectors/SdmxConnectionAdapter.java | 155 ---------------- .../main/java/internal/connectors/Util.java | 20 +-- .../connectors/{ => drivers}/AbsDriver.java | 7 +- .../{ => drivers}/EurostatDriver.java | 7 +- .../connectors/{ => drivers}/IloDriver.java | 7 +- .../connectors/{ => drivers}/ImfDriver.java | 7 +- .../connectors/drivers/InseeDriver.java | 102 +++++++++++ .../connectors/{ => drivers}/NbbDriver.java | 7 +- .../connectors/{ => drivers}/OecdDriver.java | 7 +- .../{ => drivers}/Sdmx20Driver.java | 6 +- .../{ => drivers}/Sdmx21Config.java | 3 +- .../{ => drivers}/Sdmx21Driver.java | 8 +- .../connectors/{ => drivers}/UisDriver.java | 7 +- .../connectors/{ => drivers}/WbDriver.java | 7 +- .../internal/xml/stream/InseeDataFactory.java | 96 ++++++++++ .../connectors/ConnectorsResource.java | 4 +- .../connectors/DataCursorAdapterTest.java | 4 +- .../connectors/SdmxConnectionAdapterTest.java | 2 +- 27 files changed, 580 insertions(+), 414 deletions(-) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{CachedSdmxConnection.java => CachedResource.java} (64%) create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{SdmxDriverSupport.java => ConnectorsDriverSupport.java} (76%) create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ClientSupplier.java => GenericSDMXClientSupplier.java} (92%) delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{DataCursorAdapter.java => PortableTimeSeriesCursor.java} (93%) delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/AbsDriver.java (79%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/EurostatDriver.java (79%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/IloDriver.java (79%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/ImfDriver.java (79%) create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/NbbDriver.java (80%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/OecdDriver.java (78%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/Sdmx20Driver.java (88%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/Sdmx21Config.java (96%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/Sdmx21Driver.java (95%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/UisDriver.java (79%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/{ => drivers}/WbDriver.java (81%) create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java similarity index 64% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java index 30cff6ef..2b911dbb 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java @@ -26,7 +26,6 @@ import be.nbb.sdmx.facade.util.TypedId; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import java.io.IOException; import java.time.Clock; import java.util.Map; @@ -36,16 +35,17 @@ * * @author Philippe Charles */ -final class CachedSdmxConnection extends SdmxConnectionAdapter { +final class CachedResource implements ConnectorsConnection.Resource { + private final ConnectorsConnection.Resource delegate; private final TtlCache cache; private final TypedId> dataflowsKey; private final TypedId dataflowKey; private final TypedId dataflowStructureKey; private final TypedId dataKey; - CachedSdmxConnection(GenericSDMXClient client, String host, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { - super(client); + CachedResource(ConnectorsConnection.Resource delegate, String host, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { + this.delegate = delegate; this.cache = TtlCache.of(cache, clock, ttlInMillis); String base = host + languages.toString(); this.dataflowsKey = TypedId.of("flows://" + base); @@ -55,17 +55,17 @@ final class CachedSdmxConnection extends SdmxConnectionAdapter { } @Override - protected Map loadDataFlowsById() throws IOException { + public Map loadDataFlowsById() throws IOException { Map result = cache.get(dataflowsKey); if (result == null) { - result = super.loadDataFlowsById(); + result = delegate.loadDataFlowsById(); cache.put(dataflowsKey, result); } return result; } @Override - protected Dataflow loadDataflow(DataflowRef flowRef) throws IOException { + public Dataflow loadDataflow(DataflowRef flowRef) throws IOException { // check if dataflow has been already loaded by #loadDataFlowsById Map dataFlows = cache.get(dataflowsKey); if (dataFlows != null) { @@ -78,39 +78,44 @@ protected Dataflow loadDataflow(DataflowRef flowRef) throws IOException { TypedId id = dataflowKey.with(flowRef); Dataflow result = cache.get(id); if (result == null) { - result = super.loadDataflow(flowRef); + result = delegate.loadDataflow(flowRef); cache.put(id, result); } return result; } @Override - protected DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { + public DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { TypedId id = dataflowStructureKey.with(flowRef); DataFlowStructure result = cache.get(id); if (result == null) { - result = super.loadDataStructure(flowRef); + result = delegate.loadDataStructure(flowRef); cache.put(id, result); } return result; } @Override - protected DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - if (serieskeysonly) { - TypedId id = dataKey.with(flowRef); - SdmxRepository result = cache.get(id); - if (result == null || key.supersedes(Key.parse(result.getName()))) { - try (DataCursor cursor = super.loadData(flowRef, key, true)) { - result = SdmxRepository.builder() - .copyOf(flowRef, cursor) - .name(key.toString()) - .build(); - } - cache.put(id, result); + public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + if (!serieskeysonly) { + return delegate.loadData(flowRef, key, serieskeysonly); + } + TypedId id = dataKey.with(flowRef); + SdmxRepository result = cache.get(id); + if (result == null || key.supersedes(Key.parse(result.getName()))) { + try (DataCursor cursor = delegate.loadData(flowRef, key, true)) { + result = SdmxRepository.builder() + .copyOf(flowRef, cursor) + .name(key.toString()) + .build(); } - return result.asConnection().getData(flowRef, DataQuery.of(key, true)); + cache.put(id, result); } - return super.loadData(flowRef, key, serieskeysonly); + return result.asConnection().getData(flowRef, DataQuery.of(key, true)); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return delegate.isSeriesKeysOnlySupported(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java new file mode 100644 index 00000000..9111b768 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java @@ -0,0 +1,109 @@ +/* + * Copyright 2015 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.SdmxConnection; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +final class ConnectorsConnection implements SdmxConnection { + + interface Resource { + + @Nonnull + Map loadDataFlowsById() throws IOException; + + @Nonnull + it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) throws IOException; + + @Nonnull + it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException; + + @Nonnull + DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException; + + boolean isSeriesKeysOnlySupported(); + } + + private final Resource resource; + private boolean closed; + + ConnectorsConnection(Resource resource) { + this.resource = resource; + this.closed = false; + } + + @Override + public Set getDataflows() throws IOException { + checkState(); + return resource.loadDataFlowsById().values().stream() + .map(Util::toDataflow) + .collect(Collectors.toSet()); + } + + @Override + public Dataflow getDataflow(DataflowRef flowRef) throws IOException { + checkState(); + return Util.toDataflow(resource.loadDataflow(flowRef)); + } + + @Override + public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { + checkState(); + return Util.toDataStructure(resource.loadDataStructure(flowRef)); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { + checkState(); + boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); + if (serieskeysonly && !isSeriesKeysOnlySupported()) { + throw new IllegalStateException("serieskeysonly not supported"); + } + return resource.loadData(flowRef, query.getKey(), serieskeysonly); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return resource.isSeriesKeysOnlySupported(); + } + + @Override + public void close() throws IOException { + closed = true; + } + + private void checkState() throws IOException { + if (closed) { + throw new IOException("Connection closed"); + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java similarity index 76% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java index 72171669..e3267df0 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java @@ -45,24 +45,24 @@ * @author Philippe Charles */ @ThreadSafe -final class SdmxDriverSupport implements HasCache { +public final class ConnectorsDriverSupport implements HasCache { @Nonnull - public static SdmxDriverSupport of(@Nonnull String prefix, @Nonnull Class clazz) { - return new SdmxDriverSupport(prefix, ClientSupplier.ofType(clazz), new ConcurrentHashMap(), Clock.systemDefaultZone()); + public static ConnectorsDriverSupport of(@Nonnull String prefix, @Nonnull Class clazz) { + return new ConnectorsDriverSupport(prefix, GenericSDMXClientSupplier.ofType(clazz), new ConcurrentHashMap(), Clock.systemDefaultZone()); } @Nonnull - public static SdmxDriverSupport of(@Nonnull String prefix, @Nonnull ClientSupplier supplier) { - return new SdmxDriverSupport(prefix, supplier, new ConcurrentHashMap(), Clock.systemDefaultZone()); + public static ConnectorsDriverSupport of(@Nonnull String prefix, @Nonnull GenericSDMXClientSupplier supplier) { + return new ConnectorsDriverSupport(prefix, supplier, new ConcurrentHashMap(), Clock.systemDefaultZone()); } private final String prefix; - private final ClientSupplier supplier; + private final GenericSDMXClientSupplier supplier; private final AtomicReference cache; private final Clock clock; - private SdmxDriverSupport(String prefix, ClientSupplier supplier, ConcurrentMap cache, Clock clock) { + private ConnectorsDriverSupport(String prefix, GenericSDMXClientSupplier supplier, ConcurrentMap cache, Clock clock) { this.prefix = prefix; this.supplier = supplier; this.cache = new AtomicReference<>(cache); @@ -70,14 +70,7 @@ private SdmxDriverSupport(String prefix, ClientSupplier supplier, ConcurrentMap } public SdmxConnection connect(URI uri, Map info, LanguagePriorityList languages) throws IOException { - try { - URL endpoint = new URL(uri.toString().substring(prefix.length())); - GenericSDMXClient client = supplier.getClient(endpoint, info, languages); - applyTimeouts(client, info); - return new CachedSdmxConnection(client, endpoint.getHost(), languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); - } catch (MalformedURLException ex) { - throw new IOException(ex); - } + return new ConnectorsConnection(getResource(uri, info, languages)); } public boolean acceptsURI(URI uri) throws IOException { @@ -99,7 +92,17 @@ public static Collection entry(@Nonnull String name, @Nonnull return Collections.singleton(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); } - // + private ConnectorsConnection.Resource getResource(URI uri, Map info, LanguagePriorityList languages) throws IOException { + try { + URL endpoint = new URL(uri.toString().substring(prefix.length())); + GenericSDMXClient client = supplier.getClient(endpoint, info, languages); + applyTimeouts(client, info); + return new CachedResource(new GenericSDMXClientResource(client), endpoint.getHost(), languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); + } catch (MalformedURLException ex) { + throw new IOException(ex); + } + } + private static void applyTimeouts(GenericSDMXClient client, Map info) { if (client instanceof RestSdmxClient) { ((RestSdmxClient) client).setConnectTimeout(CONNECT_TIMEOUT.get(info, DEFAULT_CONNECT_TIMEOUT)); @@ -110,5 +113,4 @@ private static void applyTimeouts(GenericSDMXClient client, Map info) { private final static int DEFAULT_CONNECT_TIMEOUT = 1000 * 60 * 2; // 2 minutes private final static int DEFAULT_READ_TIMEOUT = 1000 * 60 * 2; // 2 minutes private static final long DEFAULT_CACHE_TTL = TimeUnit.MINUTES.toMillis(5); - // } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java new file mode 100644 index 00000000..f9a23884 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java @@ -0,0 +1,160 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.NoOpCursor; +import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.util.UnexpectedIOException; +import it.bancaditalia.oss.sdmx.api.DSDIdentifier; +import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; +import it.bancaditalia.oss.sdmx.client.custom.DotStat; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import it.bancaditalia.oss.sdmx.exceptions.SdmxResponseException; +import java.io.IOException; +import java.util.Map; +import java.util.logging.Level; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor +@lombok.extern.java.Log +final class GenericSDMXClientResource implements ConnectorsConnection.Resource { + + private final GenericSDMXClient client; + + @Override + public Map loadDataFlowsById() throws IOException { + Map result; + + try { + result = client.getDataflows(); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get datasets"); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting datasets"); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting datsets"); + } + + return result; + } + + @Override + public it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) throws IOException { + it.bancaditalia.oss.sdmx.api.Dataflow result; + + try { + result = client.getDataflow(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get details from dataset '%s'", flowRef); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting details from dataset '%s'", flowRef); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting details from dataset '%s'", flowRef); + } + + return result; + } + + @Override + public it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { + it.bancaditalia.oss.sdmx.api.DSDIdentifier dsd = loadDsdIdentifier(flowRef); + + it.bancaditalia.oss.sdmx.api.DataFlowStructure result; + + try { + result = client.getDataFlowStructure(dsd, true); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get data structure from dataset '%s'", flowRef); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting data structure from dataset '%s'", flowRef); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting data structure from dataset '%s'", flowRef); + } + + return result; + } + + @Override + public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + it.bancaditalia.oss.sdmx.api.Dataflow dataflow = loadDataflow(flowRef); + it.bancaditalia.oss.sdmx.api.DataFlowStructure dfs = loadDataStructure(flowRef); + + DataCursor result; + + try { + result = client instanceof HasDataCursor + ? ((HasDataCursor) client).getDataCursor(dataflow, dfs, key, serieskeysonly) + : new PortableTimeSeriesCursor(client.getTimeSeries(dataflow, dfs, key.toString(), null, null, serieskeysonly, null, false), ObsParser.standard()); + } catch (SdmxException ex) { + if (isNoResultMatchingQuery(ex)) { + return NoOpCursor.noOp(); + } + throw expected(ex, "Failed to get data from dataset '%s' with key '%s'", flowRef, key); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting data from dataset '%s' with key '%s'", flowRef, key); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting data from dataset '%s' with key '%s'", flowRef, key); + } + + return result; + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return client instanceof HasSeriesKeysOnlySupported + && ((HasSeriesKeysOnlySupported) client).isSeriesKeysOnlySupported(); + } + + private it.bancaditalia.oss.sdmx.api.DSDIdentifier loadDsdIdentifier(DataflowRef flowRef) throws IOException { + return client instanceof DotStat + ? new DSDIdentifier(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()) + : loadDataflow(flowRef).getDsdIdentifier(); + } + + private static IOException expected(SdmxException ex, String format, Object... args) { + return new IOException(String.format(format, args), ex); + } + + private static IOException unexpected(RuntimeException ex, String format, Object... args) { + log.log(Level.WARNING, format, args); + return new UnexpectedIOException(ex); + } + + private static IOException unexpectedNull(String format, Object... args) { + String msg = String.format(format, args); + log.log(Level.WARNING, msg); + return new UnexpectedIOException(new NullPointerException(msg)); + } + + private static boolean isNoResultMatchingQuery(SdmxException ex) { + return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == 100; + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java similarity index 92% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java index adec81bf..8674da1a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ClientSupplier.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java @@ -28,13 +28,13 @@ * * @author Philippe Charles */ -interface ClientSupplier { +public interface GenericSDMXClientSupplier { @Nonnull GenericSDMXClient getClient(@Nonnull URL endpoint, @Nonnull Map info, @Nonnull LanguagePriorityList langs) throws MalformedURLException; @Nonnull - static ClientSupplier ofType(@Nonnull Class clazz) { + static GenericSDMXClientSupplier ofType(@Nonnull Class clazz) { return (URL endpoint, Map info, LanguagePriorityList langs) -> { try { RestSdmxClient result = clazz.newInstance(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java index 75875067..9aab6dc7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java @@ -28,7 +28,7 @@ * * @author Philippe Charles */ -interface HasDataCursor { +public interface HasDataCursor { @Nonnull DataCursor getDataCursor(@Nonnull Dataflow dataflow, @Nonnull DataFlowStructure dsd, @Nonnull Key resource, boolean serieskeysonly) throws SdmxException, IOException; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java index a0d4fd29..d3300d03 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java @@ -20,7 +20,7 @@ * * @author Philippe Charles */ -interface HasSeriesKeysOnlySupported { +public interface HasSeriesKeysOnlySupported { boolean isSeriesKeysOnlySupported(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java deleted file mode 100644 index b1cd2e78..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/InseeDriver.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2016 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Frequency; -import static be.nbb.sdmx.facade.Frequency.ANNUAL; -import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; -import static be.nbb.sdmx.facade.Frequency.MONTHLY; -import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import static be.nbb.sdmx.facade.Frequency.UNDEFINED; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; -import be.nbb.sdmx.facade.repo.Series; -import be.nbb.sdmx.facade.util.FreqParser; -import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.util.ObsParser; -import be.nbb.sdmx.facade.util.SafeParser; -import be.nbb.sdmx.facade.util.SdmxMediaType; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; -import it.bancaditalia.oss.sdmx.client.RestSdmxClient; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; -import java.io.IOException; -import java.io.Reader; -import java.net.URL; -import java.time.LocalDateTime; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; - -/** - * - * @author Philippe Charles - */ -@ServiceProvider(service = SdmxWebDriver.class) -public final class InseeDriver implements SdmxWebDriver, HasCache { - - private static final String PREFIX = "sdmx:insee:"; - - private final XMLInputFactory xml = XMLInputFactory.newInstance(); - - @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l, xml)); - - @Override - public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); - } - - // - private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { - - private final XMLInputFactory factory; - - private InseeClient(URL endpoint, LanguagePriorityList langs, XMLInputFactory factory) { - super("", endpoint, false, false, true); - this.languages = Util.fromLanguages(langs); - this.factory = factory; - } - - @Override - public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { - String query = buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false); - // FIXME: avoid in-memory copy - List data = runQuery((r, l) -> parse(r, Util.toDataStructure(dsd)), query, SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); - return Series.asCursor(data, resource); - } - - @Override - public boolean isSeriesKeysOnlySupported() { - return true; - } - - private List parse(Reader xmlReader, DataStructure dsd) throws XMLStreamException, SdmxException { - try { - return Series.copyOf(SdmxXmlStreams.compactData21(dsd, CustomDataParser.INSTANCE).get(factory, xmlReader)); - } catch (IOException ex) { - throw new SdmxIOException("Cannot parse compact data 21", ex); - } - } - } - - // https://www.insee.fr/fr/information/2862759 - private enum CustomDataParser implements Function { - - INSTANCE; - - private final Map> parsers = Stream.of(Frequency.values()).collect(Collectors.toMap(Function.identity(), o -> getPeriodParser(o))); - - @Override - public SdmxXmlStreams.DataParser apply(DataStructure dsd) { - FreqParser freq = getFreqParser(dsd); - Function> period = parsers::get; - return new SdmxXmlStreams.DataParser(Key.builder(dsd), freq, new ObsParser(period, SafeParser.onDouble()), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); - } - - private static FreqParser getFreqParser(DataStructure dsd) { - int index = FreqParser.getFrequencyCodeIdIndex(dsd); - return index != FreqParser.NO_FREQUENCY_CODE_ID_INDEX - ? (k, a) -> parseByFreq(k, index) - : (k, a) -> Frequency.UNDEFINED; - } - - private static Frequency parseByFreq(Key.Builder key, int index) { - String codeId = key.getItem(index); - return codeId.length() == 1 ? parseByCode(codeId.charAt(0)) : Frequency.UNDEFINED; - } - - private static Frequency parseByCode(char code) { - switch (code) { - case 'A': - return ANNUAL; - case 'S': - return HALF_YEARLY; - case 'T': - return QUARTERLY; - case 'M': - return MONTHLY; - case 'B': - return MONTHLY; - default: - return UNDEFINED; - } - } - - private static SafeParser getPeriodParser(Frequency freq) { - switch (freq) { - case ANNUAL: - return SafeParser.onDatePattern("yyyy"); - case HALF_YEARLY: - return SafeParser.onYearFreqPos("S", 2); - case QUARTERLY: - return SafeParser.onYearFreqPos("Q", 4); - case MONTHLY: - return SafeParser.onDatePattern("yyyy-MM").or(SafeParser.onYearFreqPos("B", 12)); - default: - return SafeParser.onNull(); - } - } - } - // -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java similarity index 93% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index c3428320..39549691 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/DataCursorAdapter.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -34,7 +34,7 @@ * * @author Philippe Charles */ -final class DataCursorAdapter implements DataCursor { +final class PortableTimeSeriesCursor implements DataCursor { private final Iterator data; private final ObsParser obs; @@ -43,7 +43,7 @@ final class DataCursorAdapter implements DataCursor { private boolean closed; private boolean hasObs; - DataCursorAdapter(List data, ObsParser obs) { + PortableTimeSeriesCursor(List data, ObsParser obs) { this.data = data.iterator(); this.obs = obs; this.closed = false; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java deleted file mode 100644 index 1deb2c97..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/SdmxConnectionAdapter.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.DataQueryDetail; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.util.NoOpCursor; -import be.nbb.sdmx.facade.util.ObsParser; -import it.bancaditalia.oss.sdmx.api.DSDIdentifier; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import it.bancaditalia.oss.sdmx.client.custom.DotStat; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.exceptions.SdmxResponseException; -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -class SdmxConnectionAdapter implements SdmxConnection { - - private final GenericSDMXClient client; - private boolean closed; - - public SdmxConnectionAdapter(GenericSDMXClient client) { - this.client = client; - this.closed = false; - } - - @Override - final public Set getDataflows() throws IOException { - checkState(); - Set result = new HashSet<>(); - for (it.bancaditalia.oss.sdmx.api.Dataflow o : loadDataFlowsById().values()) { - result.add(Util.toDataflow(o)); - } - return Collections.unmodifiableSet(result); - } - - @Override - final public Dataflow getDataflow(DataflowRef flowRef) throws IOException { - checkState(); - return Util.toDataflow(loadDataflow(flowRef)); - } - - @Override - final public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { - checkState(); - return Util.toDataStructure(loadDataStructure(flowRef)); - } - - @Override - final public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { - checkState(); - boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); - if (serieskeysonly && !isSeriesKeysOnlySupported()) { - throw new IllegalStateException("serieskeysonly not supported"); - } - return loadData(flowRef, query.getKey(), serieskeysonly); - } - - @Override - public boolean isSeriesKeysOnlySupported() { - return client instanceof HasSeriesKeysOnlySupported - && ((HasSeriesKeysOnlySupported) client).isSeriesKeysOnlySupported(); - } - - @Override - public void close() throws IOException { - closed = true; - } - - private void checkState() throws IOException { - if (closed) { - throw new IOException("Connection closed"); - } - } - - @Nonnull - protected Map loadDataFlowsById() throws IOException { - try { - return client.getDataflows(); - } catch (SdmxException ex) { - throw new IOException("Failed to get datasets", ex); - } - } - - @Nonnull - protected it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) throws IOException { - try { - return client.getDataflow(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()); - } catch (SdmxException ex) { - throw new IOException("Failed to get details from dataset '" + flowRef + "'", ex); - } - } - - @Nonnull - protected it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { - it.bancaditalia.oss.sdmx.api.DSDIdentifier dsd = client instanceof DotStat - ? new DSDIdentifier(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()) - : loadDataflow(flowRef).getDsdIdentifier(); - try { - return client.getDataFlowStructure(dsd, true); - } catch (SdmxException ex) { - throw new IOException("Failed to get data structure from dataset '" + flowRef + "'", ex); - } - } - - @Nonnull - protected DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - it.bancaditalia.oss.sdmx.api.Dataflow dataflow = loadDataflow(flowRef); - it.bancaditalia.oss.sdmx.api.DataFlowStructure dfs = loadDataStructure(flowRef); - try { - return client instanceof HasDataCursor - ? ((HasDataCursor) client).getDataCursor(dataflow, dfs, key, serieskeysonly) - : new DataCursorAdapter(client.getTimeSeries(dataflow, dfs, key.toString(), null, null, serieskeysonly, null, false), ObsParser.standard()); - } catch (SdmxException ex) { - if (isNoResultMatchingQuery(ex)) { - return NoOpCursor.noOp(); - } - throw new IOException("Failed to get data from dataset '" + flowRef + "' with key '" + key + "'", ex); - } - } - - // - private static boolean isNoResultMatchingQuery(SdmxException ex) { - return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == 100; - } - // -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 65b23cb4..b5b874b3 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -32,17 +32,17 @@ * @author Philippe Charles */ @lombok.experimental.UtilityClass -class Util { +public class Util { - be.nbb.sdmx.facade.Dataflow toDataflow(Dataflow dataflow) { + public be.nbb.sdmx.facade.Dataflow toDataflow(Dataflow dataflow) { return be.nbb.sdmx.facade.Dataflow.of(DataflowRef.parse(dataflow.getFullIdentifier()), toDataStructureRef(dataflow.getDsdIdentifier()), dataflow.getDescription()); } - be.nbb.sdmx.facade.DataStructureRef toDataStructureRef(DSDIdentifier input) { + public be.nbb.sdmx.facade.DataStructureRef toDataStructureRef(DSDIdentifier input) { return DataStructureRef.of(input.getAgency(), input.getId(), input.getVersion()); } - be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { + public be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { be.nbb.sdmx.facade.Dimension.Builder result = be.nbb.sdmx.facade.Dimension.builder() .id(o.getId()) .position(o.getPosition()); @@ -54,7 +54,7 @@ be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { return result.build(); } - DataStructure toDataStructure(DataFlowStructure dfs) { + public DataStructure toDataStructure(DataFlowStructure dfs) { DataStructure.Builder result = DataStructure.builder() .ref(DataStructureRef.of(dfs.getAgency(), dfs.getId(), dfs.getVersion())) .label(getNonNullName(dfs)) @@ -64,7 +64,7 @@ DataStructure toDataStructure(DataFlowStructure dfs) { return result.build(); } - it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.sdmx.facade.LanguagePriorityList l) { + public it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.sdmx.facade.LanguagePriorityList l) { return it.bancaditalia.oss.sdmx.util.LanguagePriorityList.parse(l.toString()); } @@ -75,8 +75,8 @@ private String getNonNullName(DataFlowStructure dfs) { return result != null ? result : dfs.getId(); } - static final BoolProperty SUPPORTS_COMPRESSION = new BoolProperty("supportsCompression"); - static final BoolProperty NEEDS_CREDENTIALS = new BoolProperty("needsCredentials"); - static final BoolProperty NEEDS_URL_ENCODING = new BoolProperty("needsURLEncoding"); - static final BoolProperty SERIES_KEYS_ONLY_SUPPORTED = new BoolProperty("seriesKeysOnlySupported"); + public static final BoolProperty SUPPORTS_COMPRESSION = new BoolProperty("supportsCompression"); + public static final BoolProperty NEEDS_CREDENTIALS = new BoolProperty("needsCredentials"); + public static final BoolProperty NEEDS_URL_ENCODING = new BoolProperty("needsURLEncoding"); + public static final BoolProperty SERIES_KEYS_ONLY_SUPPORTED = new BoolProperty("seriesKeysOnlySupported"); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java similarity index 79% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java index d95d30fa..dfdfb91d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class AbsDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:abs:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ABS.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, ABS.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx"); + return ConnectorsDriverSupport.entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java similarity index 79% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java index 5baf5c04..3dd99cb6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class EurostatDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:eurostat:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, EUROSTAT.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, EUROSTAT.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest"); + return ConnectorsDriverSupport.entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java similarity index 79% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java index bb7b5195..310d4979 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/IloDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class IloDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:ilo:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, ILO.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, ILO.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest"); + return ConnectorsDriverSupport.entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java similarity index 79% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java index 9fd59b8a..22e81b08 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class ImfDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:imf:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, IMF.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, IMF.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx"); + return ConnectorsDriverSupport.entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java new file mode 100644 index 00000000..25edeb0b --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -0,0 +1,102 @@ +/* + * Copyright 2016 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.util.SdmxMediaType; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import it.bancaditalia.oss.sdmx.api.DataFlowStructure; +import it.bancaditalia.oss.sdmx.api.Dataflow; +import it.bancaditalia.oss.sdmx.client.RestSdmxClient; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; +import java.io.IOException; +import java.io.Reader; +import java.net.URL; +import java.util.Collection; +import java.util.List; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import org.openide.util.lookup.ServiceProvider; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.HasDataCursor; +import internal.connectors.HasSeriesKeysOnlySupported; +import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.Util; +import internal.xml.stream.InseeDataFactory; + +/** + * + * @author Philippe Charles + */ +@ServiceProvider(service = SdmxWebDriver.class) +public final class InseeDriver implements SdmxWebDriver, HasCache { + + private static final String PREFIX = "sdmx:insee:"; + + private final XMLInputFactory xml = XMLInputFactory.newInstance(); + + @lombok.experimental.Delegate + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l, xml)); + + @Override + public Collection getDefaultEntryPoints() { + return ConnectorsDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); + } + + // + private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { + + private final XMLInputFactory xmlFactory; + private final InseeDataFactory dataFactory; + + private InseeClient(URL endpoint, LanguagePriorityList langs, XMLInputFactory xmlFactory) { + super("", endpoint, false, false, true); + this.languages = Util.fromLanguages(langs); + this.xmlFactory = xmlFactory; + this.dataFactory = new InseeDataFactory(); + } + + @Override + public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { + String query = buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false); + // FIXME: avoid in-memory copy + List data = runQuery((r, l) -> parse(r, Util.toDataStructure(dsd)), query, SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); + return Series.asCursor(data, resource); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return true; + } + + private List parse(Reader xmlReader, DataStructure dsd) throws XMLStreamException, SdmxException { + try { + return Series.copyOf(SdmxXmlStreams.compactData21(dsd, dataFactory).get(xmlFactory, xmlReader)); + } catch (IOException ex) { + throw new SdmxIOException("Cannot parse compact data 21", ex); + } + } + } + // +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java similarity index 80% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java index 886d8fab..41910fd8 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class NbbDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:nbb:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, NBB.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, NBB.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx"); + return ConnectorsDriverSupport.entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java similarity index 78% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java index 099b1286..81f3b541 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class OecdDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:oecd:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, OECD.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, OECD.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx"); + return ConnectorsDriverSupport.entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java similarity index 88% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java index 0a3e3290..fc27b856 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.LanguagePriorityList; import static internal.connectors.Util.NEEDS_CREDENTIALS; @@ -27,6 +27,8 @@ import java.util.Map; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.Util; /** * @@ -38,7 +40,7 @@ public final class Sdmx20Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx20:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, Sdmx20Client::new); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, Sdmx20Client::new); @Override public List getDefaultEntryPoints() { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Config.java similarity index 96% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Config.java index 55b889ae..7fdbdd09 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Config.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Config.java @@ -14,8 +14,9 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; +import internal.connectors.Util; import java.util.Map; /** diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java similarity index 95% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index c3e5e561..6414113c 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -41,6 +41,10 @@ import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.HasDataCursor; +import internal.connectors.HasSeriesKeysOnlySupported; +import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.Util; /** * @@ -54,7 +58,7 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private final XMLInputFactory xml = XMLInputFactory.newInstance(); @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l, xml)); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l, xml)); @Override public List getDefaultEntryPoints() { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java similarity index 79% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java index adb3b868..f4bff6b9 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/UisDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class UisDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:uis:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, UIS.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, UIS.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx"); + return ConnectorsDriverSupport.entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java similarity index 81% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java index b428f141..7cf760d7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/WbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.connectors.drivers; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -22,6 +22,7 @@ import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; /** * @@ -33,10 +34,10 @@ public final class WbDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:wb:"; @lombok.experimental.Delegate - private final SdmxDriverSupport support = SdmxDriverSupport.of(PREFIX, WB.class); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, WB.class); @Override public Collection getDefaultEntryPoints() { - return SdmxDriverSupport.entry("WB", "World Bank", "sdmx:wb:http://api.worldbank.org"); + return ConnectorsDriverSupport.entry("WB", "World Bank", "sdmx:wb:http://api.worldbank.org"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java new file mode 100644 index 00000000..d79c97c9 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java @@ -0,0 +1,96 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.xml.stream; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Frequency; +import static be.nbb.sdmx.facade.Frequency.ANNUAL; +import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; +import static be.nbb.sdmx.facade.Frequency.MONTHLY; +import static be.nbb.sdmx.facade.Frequency.QUARTERLY; +import static be.nbb.sdmx.facade.Frequency.UNDEFINED; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import java.time.LocalDateTime; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * https://www.insee.fr/fr/information/2862759 + * + * @author Philippe Charles + */ +public final class InseeDataFactory implements Function { + + private final Map> parsers = Stream.of(Frequency.values()).collect(Collectors.toMap(Function.identity(), o -> getPeriodParser(o))); + + @Override + public SdmxXmlStreams.DataParser apply(DataStructure dsd) { + FreqParser freq = getFreqParser(dsd); + Function> period = parsers::get; + return new SdmxXmlStreams.DataParser(Key.builder(dsd), freq, new ObsParser(period, SafeParser.onDouble()), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + } + + private static FreqParser getFreqParser(DataStructure dsd) { + int index = FreqParser.getFrequencyCodeIdIndex(dsd); + return index != FreqParser.NO_FREQUENCY_CODE_ID_INDEX + ? (k, a) -> parseByFreq(k, index) + : (k, a) -> Frequency.UNDEFINED; + } + + private static Frequency parseByFreq(Key.Builder key, int index) { + String codeId = key.getItem(index); + return codeId.length() == 1 ? parseByCode(codeId.charAt(0)) : Frequency.UNDEFINED; + } + + private static Frequency parseByCode(char code) { + switch (code) { + case 'A': + return ANNUAL; + case 'S': + return HALF_YEARLY; + case 'T': + return QUARTERLY; + case 'M': + return MONTHLY; + case 'B': + return MONTHLY; + default: + return UNDEFINED; + } + } + + private static SafeParser getPeriodParser(Frequency freq) { + switch (freq) { + case ANNUAL: + return SafeParser.onDatePattern("yyyy"); + case HALF_YEARLY: + return SafeParser.onYearFreqPos("S", 2); + case QUARTERLY: + return SafeParser.onYearFreqPos("Q", 4); + case MONTHLY: + return SafeParser.onDatePattern("yyyy-MM").or(SafeParser.onYearFreqPos("B", 12)); + default: + return SafeParser.onNull(); + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index e3f4e391..d453cb6a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -62,7 +62,7 @@ public SdmxRepository nbb() throws IOException { return SdmxRepository.builder() .dataStructures(structs.stream().map(Util::toDataStructure).collect(Collectors.toList())) .dataflows(flows.stream().map(Util::toDataflow).collect(Collectors.toList())) - .copyOf(ref, new DataCursorAdapter(data, ObsParser.standard())) + .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("NBB") .seriesKeysOnlySupported(false) .build(); @@ -81,7 +81,7 @@ public SdmxRepository ecb() throws IOException { return SdmxRepository.builder() .dataStructures(structs.stream().map(Util::toDataStructure).collect(Collectors.toList())) .dataflows(flows.stream().map(Util::toDataflow).collect(Collectors.toList())) - .copyOf(ref, new DataCursorAdapter(data, ObsParser.standard())) + .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("ECB") .seriesKeysOnlySupported(true) .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java index 1f8a1119..499133fb 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java @@ -50,7 +50,7 @@ public static void beforeClass() throws IOException { @Test public void test() throws IOException { - try (DataCursorAdapter cursor = new DataCursorAdapter(DATA, ObsParser.standard())) { + try (PortableTimeSeriesCursor cursor = new PortableTimeSeriesCursor(DATA, ObsParser.standard())) { assertThat(Series.copyOf(cursor)) .hasSize(120) .allMatch(o -> o.getFrequency().equals(Frequency.ANNUAL)) @@ -71,6 +71,6 @@ public void test() throws IOException { @Test public void testCompliance() { - DataCursorAssert.assertCompliance(() -> new DataCursorAdapter(DATA, ObsParser.standard())); + DataCursorAssert.assertCompliance(() -> new PortableTimeSeriesCursor(DATA, ObsParser.standard())); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java index c201a019..d397f54f 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java @@ -29,6 +29,6 @@ public class SdmxConnectionAdapterTest { @Test public void testCompliance() { DataflowRef ref = DataflowRef.parse("XYZ"); - ConnectionAssert.assertCompliance(() -> new SdmxConnectionAdapter(null), ref); + ConnectionAssert.assertCompliance(() -> new ConnectorsConnection(null), ref); } } From ad549884a1572aaad35b12c6009b9ecc03e4dd55 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 19 Oct 2017 11:53:28 +0200 Subject: [PATCH 13/68] Refactored xmlstream parsers. --- .../be/nbb/sdmx/facade/util/DataFactory.java | 48 ++++++++++ .../be/nbb/sdmx/facade/util/FreqParser.java | 36 +++---- .../be/nbb/sdmx/facade/util/FreqUtil.java | 30 ++++++ .../be/nbb/sdmx/facade/util/ObsParser.java | 2 +- .../be/nbb/sdmx/facade/util/SafeParser.java | 2 +- .../facade/xml/stream/GenericDataParser.java | 77 --------------- .../facade/xml/stream/SdmxXmlStreams.java | 78 +++------------ .../nbb/sdmx/facade/xml/stream/XMLStream.java | 6 +- .../stream/XMLStreamGenericDataCursor.java | 68 +++++++++++-- .../main/java/internal/util/FreqParsers.java | 10 +- .../internal/util/StandardDataFactory.java | 62 ++++++++++++ .../XMLStreamGenericDataCursorTest.java | 12 +-- .../connectors/drivers/InseeDriver.java | 2 +- .../util/drivers/InseeDataFactory.java | 93 ++++++++++++++++++ .../internal/xml/stream/InseeDataFactory.java | 96 ------------------- 15 files changed, 343 insertions(+), 279 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/GenericDataParser.java create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java new file mode 100644 index 00000000..7428b03f --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java @@ -0,0 +1,48 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Key; +import internal.util.StandardDataFactory; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +public interface DataFactory { + + @Nonnull + Key.Builder getKeyBuilder(@Nonnull DataStructure dsd); + + @Nonnull + FreqParser getFreqParser(@Nonnull DataStructure dsd); + + @Nonnull + ObsParser getObsParser(@Nonnull DataStructure dsd); + + @Nonnull + static DataFactory sdmx20() { + return StandardDataFactory.SDMX20; + } + + @Nonnull + static DataFactory sdmx21() { + return StandardDataFactory.SDMX21; + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java index 4bd078ff..a9c4de69 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java @@ -17,10 +17,10 @@ package be.nbb.sdmx.facade.util; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; import internal.util.FreqParsers; +import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; @@ -35,31 +35,33 @@ public interface FreqParser { @Nonnull static FreqParser sdmx20() { - return (k, a) -> FreqParsers.parseByTimeFormat(a); + return FreqParsers::parseSdmx20; } @Nonnull - static FreqParser sdmx21(@Nonnull DataStructure dfs) { - return sdmx21(getFrequencyCodeIdIndex(dfs)); + static FreqParser sdmx21(@Nonnull DataStructure dsd) { + return of(FreqUtil.extractorByIndex(dsd), FreqUtil::parseByFreq); } @Nonnull static FreqParser sdmx21(int frequencyCodeIdIndex) { - return frequencyCodeIdIndex != NO_FREQUENCY_CODE_ID_INDEX - ? (k, a) -> FreqParsers.parseByFreq(k, frequencyCodeIdIndex) - : (k, a) -> Frequency.UNDEFINED; + return of(FreqUtil.extractorByIndex(frequencyCodeIdIndex), FreqUtil::parseByFreq); } - static final int NO_FREQUENCY_CODE_ID_INDEX = -1; - - static int getFrequencyCodeIdIndex(@Nonnull DataStructure dfs) { - for (Dimension o : dfs.getDimensions()) { - switch (o.getId()) { - case FreqUtil.FREQ_CONCEPT: - case "FREQUENCY": - return (o.getPosition() - 1); + @Nonnull + static FreqParser of( + @Nonnull BiFunction, String> extractor, + @Nonnull Function mapper) { + return (k, a) -> { + String code = extractor.apply(k, a); + if (code == null) { + return Frequency.UNDEFINED; + } + Frequency freq = mapper.apply(code); + if (freq == null) { + return Frequency.UNDEFINED; } - } - return NO_FREQUENCY_CODE_ID_INDEX; + return freq; + }; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java index 52d8be8a..f7fabbb6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java @@ -16,6 +16,8 @@ */ package be.nbb.sdmx.facade.util; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Frequency; import static be.nbb.sdmx.facade.Frequency.DAILY; import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; @@ -28,6 +30,9 @@ import javax.annotation.Nonnull; import static be.nbb.sdmx.facade.Frequency.ANNUAL; import static be.nbb.sdmx.facade.Frequency.DAILY_BUSINESS; +import be.nbb.sdmx.facade.Key; +import java.util.function.BiFunction; +import java.util.function.Function; /** * @@ -39,6 +44,31 @@ public class FreqUtil { public final String FREQ_CONCEPT = "FREQ"; public final String TIME_FORMAT_CONCEPT = "TIME_FORMAT"; + public static final int NO_FREQUENCY_CODE_ID_INDEX = -1; + + public int getFrequencyCodeIdIndex(@Nonnull DataStructure dsd) { + for (Dimension o : dsd.getDimensions()) { + switch (o.getId()) { + case FREQ_CONCEPT: + case "FREQUENCY": + return (o.getPosition() - 1); + } + } + return NO_FREQUENCY_CODE_ID_INDEX; + } + + @Nonnull + public BiFunction, String> extractorByIndex(@Nonnull DataStructure dsd) { + return extractorByIndex(getFrequencyCodeIdIndex(dsd)); + } + + @Nonnull + public BiFunction, String> extractorByIndex(int frequencyCodeIdIndex) { + return frequencyCodeIdIndex != NO_FREQUENCY_CODE_ID_INDEX + ? (k, a) -> k.getItem(frequencyCodeIdIndex) + : (k, a) -> null; + } + /** * * @param code diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java index 23dcbf60..c2488119 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java @@ -30,7 +30,7 @@ public final class ObsParser { @Nonnull public static ObsParser standard() { - return new ObsParser(SafeParser::onStandardFreq, SafeParser.onDouble()); + return new ObsParser(SafeParser::onStandardFreq, SafeParser.onStandardDouble()); } private final Function> toPeriodParser; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java index 557fc008..f16fee64 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java @@ -78,7 +78,7 @@ static SafeParser onStandardFreq(@Nonnull Frequency freq) { } @Nonnull - static SafeParser onDouble() { + static SafeParser onStandardDouble() { return SafeParsers::doubleOrNull; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/GenericDataParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/GenericDataParser.java deleted file mode 100644 index 8fa8697d..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/GenericDataParser.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.xml.stream; - -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import javax.annotation.Nonnull; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; - -/** - * - * @author Philippe Charles - */ -interface GenericDataParser { - - void parseValueElement(@Nonnull XMLStreamReader r, @Nonnull BiConsumer c) throws XMLStreamException; - - void parseTimeElement(@Nonnull XMLStreamReader r, @Nonnull Consumer c) throws XMLStreamException; - - @Nonnull - String getTimeELement(); - - @Nonnull - static GenericDataParser sdmx20() { - return new GenericDataParser() { - @Override - public void parseValueElement(XMLStreamReader r, BiConsumer c) throws XMLStreamException { - c.accept(r.getAttributeValue(null, "concept"), r.getAttributeValue(null, "value")); - } - - @Override - public void parseTimeElement(XMLStreamReader r, Consumer c) throws XMLStreamException { - c.accept(r.getElementText()); - } - - @Override - public String getTimeELement() { - return "Time"; - } - }; - } - - @Nonnull - static GenericDataParser sdmx21() { - return new GenericDataParser() { - @Override - public void parseValueElement(XMLStreamReader r, BiConsumer c) throws XMLStreamException { - c.accept(r.getAttributeValue(null, "id"), r.getAttributeValue(null, "value")); - } - - @Override - public void parseTimeElement(XMLStreamReader r, Consumer c) throws XMLStreamException { - c.accept(r.getAttributeValue(null, "value")); - } - - @Override - public String getTimeELement() { - return "ObsDimension"; - } - }; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index 66e1415e..4742f4aa 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -16,19 +16,16 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.sdmx.facade.util.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.util.ObsParser; import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.util.FreqParser; import be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.ReaderFunc; -import java.util.function.Function; /** * @@ -39,62 +36,42 @@ public class SdmxXmlStreams { @Nonnull public XMLStream compactData20(@Nonnull DataStructure dsd) throws IOException { - return compactData20(dsd, DataParser::sdmx20); + return compactData20(dsd, DataFactory.sdmx20()); } @Nonnull - public XMLStream compactData20(@Nonnull DataStructure dsd, @Nonnull Function customizer) throws IOException { - return o -> compactData20(o, customizer.apply(dsd)); - } - - @Nonnull - private DataCursor compactData20(@Nonnull XMLStreamReader r, @Nonnull DataParser p) throws IOException { - return new XMLStreamCompactDataCursor(r, p.getKeyBuilder(), p.getObsParser(), p.getFreqParser(), p.getTimeDimensionId(), p.getPrimaryMeasureId()); + public XMLStream compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return o -> new XMLStreamCompactDataCursor(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); } @Nonnull public XMLStream compactData21(@Nonnull DataStructure dsd) throws IOException { - return compactData21(dsd, DataParser::sdmx21); - } - - @Nonnull - public XMLStream compactData21(@Nonnull DataStructure dsd, @Nonnull Function customizer) throws IOException { - return o -> compactData21(o, customizer.apply(dsd)); + return compactData21(dsd, DataFactory.sdmx21()); } @Nonnull - private DataCursor compactData21(@Nonnull XMLStreamReader r, @Nonnull DataParser p) throws IOException { - return new XMLStreamCompactDataCursor(r, p.getKeyBuilder(), p.getObsParser(), p.getFreqParser(), p.getTimeDimensionId(), p.getPrimaryMeasureId()); + public XMLStream compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return o -> new XMLStreamCompactDataCursor(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull public XMLStream genericData20(@Nonnull DataStructure dsd) throws IOException { - return genericData20(dsd, DataParser::sdmx20); + return genericData20(dsd, DataFactory.sdmx20()); } @Nonnull - public XMLStream genericData20(@Nonnull DataStructure dsd, @Nonnull Function customizer) throws IOException { - return o -> genericData20(o, customizer.apply(dsd)); - } - - @Nonnull - private DataCursor genericData20(@Nonnull XMLStreamReader r, @Nonnull DataParser p) throws IOException { - return new XMLStreamGenericDataCursor(r, p.getKeyBuilder(), p.getObsParser(), p.getFreqParser(), GenericDataParser.sdmx20()); + public XMLStream genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return o -> XMLStreamGenericDataCursor.sdmx20(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull public XMLStream genericData21(@Nonnull DataStructure dsd) throws IOException { - return genericData21(dsd, DataParser::sdmx21); - } - - @Nonnull - public XMLStream genericData21(@Nonnull DataStructure dsd, @Nonnull Function customizer) throws IOException { - return o -> genericData21(o, customizer.apply(dsd)); + return genericData21(dsd, DataFactory.sdmx21()); } @Nonnull - private DataCursor genericData21(@Nonnull XMLStreamReader r, @Nonnull DataParser p) throws IOException { - return new XMLStreamGenericDataCursor(r, p.getKeyBuilder(), p.getObsParser(), p.getFreqParser(), GenericDataParser.sdmx21()); + public XMLStream genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return o -> XMLStreamGenericDataCursor.sdmx21(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull @@ -115,33 +92,4 @@ private List struct(@Nonnull XMLStreamReader reader, @Nonnull Rea throw new IOException(ex); } } - - @lombok.Value - public static class DataParser { - - @lombok.NonNull - Key.Builder keyBuilder; - - @lombok.NonNull - FreqParser freqParser; - - @lombok.NonNull - ObsParser obsParser; - - @lombok.NonNull - String timeDimensionId; - - @lombok.NonNull - String primaryMeasureId; - - @Nonnull - public static DataParser sdmx20(@Nonnull DataStructure o) { - return new DataParser(Key.builder(o), FreqParser.sdmx20(), ObsParser.standard(), "", ""); - } - - @Nonnull - public static DataParser sdmx21(@Nonnull DataStructure o) { - return new DataParser(Key.builder(o), FreqParser.sdmx21(o), ObsParser.standard(), o.getTimeDimensionId(), o.getPrimaryMeasureId()); - } - } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java index 3025a611..6a0cdb54 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java @@ -40,11 +40,15 @@ default T get(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset @Nonnull default T get(@Nonnull XMLInputFactory xf, @Nonnull Reader stream) throws IOException { + XMLStreamReader reader; + try { - return get(xf.createXMLStreamReader(stream)); + reader = xf.createXMLStreamReader(stream); } catch (XMLStreamException ex) { throw new IOException("Failed to create XML reader", ex); } + + return get(reader); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 7cc311d3..90f1c788 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -30,6 +30,9 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.util.FreqParser; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import javax.annotation.Nonnull; /** * @@ -37,6 +40,14 @@ */ final class XMLStreamGenericDataCursor implements DataCursor { + static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + return new XMLStreamGenericDataCursor(reader, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX20); + } + + static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + return new XMLStreamGenericDataCursor(reader, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX21); + } + private static final String DATASET_ELEMENT = "DataSet"; private static final String SERIES_ELEMENT = "Series"; private static final String OBS_ELEMENT = "Obs"; @@ -51,18 +62,18 @@ final class XMLStreamGenericDataCursor implements DataCursor { private final AttributesBuilder attributesBuilder; private final ObsParser obsParser; private final FreqParser freqParser; - private final GenericDataParser genericParser; + private final SeriesHeadParser headParser; private boolean closed; private boolean hasSeries; private boolean hasObs; - XMLStreamGenericDataCursor(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, GenericDataParser genericParser) { + private XMLStreamGenericDataCursor(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { this.reader = reader; this.keyBuilder = keyBuilder; this.attributesBuilder = new AttributesBuilder(); this.obsParser = obsParser; this.freqParser = freqParser; - this.genericParser = genericParser; + this.headParser = headParser; this.closed = false; this.hasSeries = false; this.hasObs = false; @@ -225,12 +236,12 @@ private Status onAttributes(boolean start, String localName) throws XMLStreamExc } private Status parseSeriesKeyValue() throws XMLStreamException { - genericParser.parseValueElement(reader, keyBuilder::put); + headParser.parseValueElement(reader, keyBuilder::put); return CONTINUE; } private Status parseAttributesValue() throws XMLStreamException { - genericParser.parseValueElement(reader, attributesBuilder::put); + headParser.parseValueElement(reader, attributesBuilder::put); return CONTINUE; } @@ -252,7 +263,7 @@ private Status onSeriesBody(boolean start, String localName) throws XMLStreamExc private Status onObs(boolean start, String localName) throws XMLStreamException { if (start) { - if (localName.equals(genericParser.getTimeELement())) { + if (localName.equals(headParser.getTimeELement())) { return parseObsTime(); } return localName.equals(OBS_VALUE_ELEMENT) ? parseObsValue() : CONTINUE; @@ -267,7 +278,7 @@ private Status parseObs() throws XMLStreamException { } private Status parseObsTime() throws XMLStreamException { - genericParser.parseTimeElement(reader, obsParser::period); + headParser.parseTimeElement(reader, obsParser::period); return CONTINUE; } @@ -279,4 +290,47 @@ private Status parseObsValue() { private boolean nextWhile(XMLStreamUtil.Func func) throws XMLStreamException { return XMLStreamUtil.nextWhile(reader, func); } + + private enum SeriesHeadParser { + + SDMX20 { + @Override + public void parseValueElement(XMLStreamReader r, BiConsumer c) throws XMLStreamException { + c.accept(r.getAttributeValue(null, "concept"), r.getAttributeValue(null, "value")); + } + + @Override + public void parseTimeElement(XMLStreamReader r, Consumer c) throws XMLStreamException { + c.accept(r.getElementText()); + } + + @Override + public String getTimeELement() { + return "Time"; + } + }, + SDMX21 { + @Override + public void parseValueElement(XMLStreamReader r, BiConsumer c) throws XMLStreamException { + c.accept(r.getAttributeValue(null, "id"), r.getAttributeValue(null, "value")); + } + + @Override + public void parseTimeElement(XMLStreamReader r, Consumer c) throws XMLStreamException { + c.accept(r.getAttributeValue(null, "value")); + } + + @Override + public String getTimeELement() { + return "ObsDimension"; + } + }; + + abstract void parseValueElement(@Nonnull XMLStreamReader r, @Nonnull BiConsumer c) throws XMLStreamException; + + abstract void parseTimeElement(@Nonnull XMLStreamReader r, @Nonnull Consumer c) throws XMLStreamException; + + @Nonnull + abstract String getTimeELement(); + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java index fb31000f..fc5d86c7 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java @@ -28,12 +28,8 @@ @lombok.experimental.UtilityClass public class FreqParsers { - public Frequency parseByTimeFormat(Function attributes) { - String value = attributes.apply(FreqUtil.TIME_FORMAT_CONCEPT); - return value != null ? FreqUtil.parseByTimeFormat(value) : Frequency.UNDEFINED; - } - - public Frequency parseByFreq(Key.Builder key, int index) { - return FreqUtil.parseByFreq(key.getItem(index)); + public Frequency parseSdmx20(Key.Builder key, Function attributes) { + String code = attributes.apply(FreqUtil.TIME_FORMAT_CONCEPT); + return code != null ? FreqUtil.parseByTimeFormat(code) : Frequency.UNDEFINED; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java new file mode 100644 index 00000000..3e68744b --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java @@ -0,0 +1,62 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.DataFactory; +import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.util.ObsParser; + +/** + * + * @author Philippe Charles + */ +public enum StandardDataFactory implements DataFactory { + + SDMX20 { + @Override + public Key.Builder getKeyBuilder(DataStructure o) { + return Key.builder(o); + } + + @Override + public FreqParser getFreqParser(DataStructure o) { + return FreqParser.sdmx20(); + } + + @Override + public ObsParser getObsParser(DataStructure o) { + return ObsParser.standard(); + } + }, SDMX21 { + @Override + public Key.Builder getKeyBuilder(DataStructure o) { + return Key.builder(o); + } + + @Override + public FreqParser getFreqParser(DataStructure o) { + return FreqParser.sdmx21(o); + } + + @Override + public ObsParser getObsParser(DataStructure o) { + return ObsParser.standard(); + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 13ccc4ed..cb1ea758 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -39,9 +39,9 @@ public void testGenericData20() throws Exception { ByteSource xml = SdmxSource.NBB_DATA; Key.Builder builder = Key.builder("SUBJECT", "LOCATION", "FREQUENCY"); - DataCursorAssert.assertCompliance(() -> new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20(), GenericDataParser.sdmx20())); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20())); - try (DataCursor o = new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20(), GenericDataParser.sdmx20())) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20())) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -83,9 +83,9 @@ public void testGenericData21() throws Exception { ByteSource xml = SdmxSource.OTHER_GENERIC21; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), GenericDataParser.sdmx21())); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))); - try (DataCursor o = new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), GenericDataParser.sdmx21())) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -119,9 +119,9 @@ public void testGenericData21Bis() throws Exception { ByteSource xml = SdmxSource.ECB_DATA; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), GenericDataParser.sdmx21())); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))); - try (DataCursor o = new XMLStreamGenericDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), GenericDataParser.sdmx21())) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 25edeb0b..22a0c574 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -43,7 +43,7 @@ import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; -import internal.xml.stream.InseeDataFactory; +import internal.util.drivers.InseeDataFactory; /** * diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java new file mode 100644 index 00000000..1585a791 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java @@ -0,0 +1,93 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util.drivers; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Frequency; +import static be.nbb.sdmx.facade.Frequency.ANNUAL; +import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; +import static be.nbb.sdmx.facade.Frequency.MONTHLY; +import static be.nbb.sdmx.facade.Frequency.QUARTERLY; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.util.DataFactory; +import be.nbb.sdmx.facade.util.FreqUtil; +import java.time.LocalDateTime; + +/** + * https://www.insee.fr/fr/information/2862759 + * + * @author Philippe Charles + */ +public final class InseeDataFactory implements DataFactory { + + @Override + public Key.Builder getKeyBuilder(DataStructure dsd) { + return Key.builder(dsd); + } + + @Override + public FreqParser getFreqParser(DataStructure dsd) { + return FreqParser.of(FreqUtil.extractorByIndex(dsd), InseeDataFactory::parseInseeFreq); + } + + @Override + public ObsParser getObsParser(DataStructure dsd) { + return new ObsParser(InseeDataFactory::onInseeTimePeriod, SafeParser.onStandardDouble()); + } + + private static Frequency parseInseeFreq(String code) { + if (code.length() == 1) { + switch (code.charAt(0)) { + case 'A': + return ANNUAL; + case 'S': + return HALF_YEARLY; + case 'T': + return QUARTERLY; + case 'M': + return MONTHLY; + case 'B': + return MONTHLY; + } + } + return Frequency.UNDEFINED; + } + + private static SafeParser onInseeTimePeriod(Frequency freq) { + switch (freq) { + case ANNUAL: + return ANNUAL_PARSER; + case HALF_YEARLY: + return HALF_YEARLY_PARSER; + case QUARTERLY: + return QUARTERLY_PARSER; + case MONTHLY: + return MONTHLY_PARSER; + default: + return DEFAULT_PARSER; + } + } + + private static final SafeParser ANNUAL_PARSER = SafeParser.onDatePattern("yyyy"); + private static final SafeParser HALF_YEARLY_PARSER = SafeParser.onYearFreqPos("S", 2); + private static final SafeParser QUARTERLY_PARSER = SafeParser.onYearFreqPos("Q", 4); + private static final SafeParser MONTHLY_PARSER = SafeParser.onDatePattern("yyyy-MM").or(SafeParser.onYearFreqPos("B", 12)); + private static final SafeParser DEFAULT_PARSER = SafeParser.onNull(); +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java deleted file mode 100644 index d79c97c9..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/xml/stream/InseeDataFactory.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.xml.stream; - -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Frequency; -import static be.nbb.sdmx.facade.Frequency.ANNUAL; -import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; -import static be.nbb.sdmx.facade.Frequency.MONTHLY; -import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import static be.nbb.sdmx.facade.Frequency.UNDEFINED; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.FreqParser; -import be.nbb.sdmx.facade.util.ObsParser; -import be.nbb.sdmx.facade.util.SafeParser; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import java.time.LocalDateTime; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * https://www.insee.fr/fr/information/2862759 - * - * @author Philippe Charles - */ -public final class InseeDataFactory implements Function { - - private final Map> parsers = Stream.of(Frequency.values()).collect(Collectors.toMap(Function.identity(), o -> getPeriodParser(o))); - - @Override - public SdmxXmlStreams.DataParser apply(DataStructure dsd) { - FreqParser freq = getFreqParser(dsd); - Function> period = parsers::get; - return new SdmxXmlStreams.DataParser(Key.builder(dsd), freq, new ObsParser(period, SafeParser.onDouble()), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); - } - - private static FreqParser getFreqParser(DataStructure dsd) { - int index = FreqParser.getFrequencyCodeIdIndex(dsd); - return index != FreqParser.NO_FREQUENCY_CODE_ID_INDEX - ? (k, a) -> parseByFreq(k, index) - : (k, a) -> Frequency.UNDEFINED; - } - - private static Frequency parseByFreq(Key.Builder key, int index) { - String codeId = key.getItem(index); - return codeId.length() == 1 ? parseByCode(codeId.charAt(0)) : Frequency.UNDEFINED; - } - - private static Frequency parseByCode(char code) { - switch (code) { - case 'A': - return ANNUAL; - case 'S': - return HALF_YEARLY; - case 'T': - return QUARTERLY; - case 'M': - return MONTHLY; - case 'B': - return MONTHLY; - default: - return UNDEFINED; - } - } - - private static SafeParser getPeriodParser(Frequency freq) { - switch (freq) { - case ANNUAL: - return SafeParser.onDatePattern("yyyy"); - case HALF_YEARLY: - return SafeParser.onYearFreqPos("S", 2); - case QUARTERLY: - return SafeParser.onYearFreqPos("Q", 4); - case MONTHLY: - return SafeParser.onDatePattern("yyyy-MM").or(SafeParser.onYearFreqPos("B", 12)); - default: - return SafeParser.onNull(); - } - } -} From 8a97ce80bd80ee545c1196c32bee0933122cc174 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 19 Oct 2017 16:24:34 +0200 Subject: [PATCH 14/68] Fixed cache issues. --- .../nbb/sdmx/facade/repo/SdmxRepository.java | 15 +- .../java/be/nbb/sdmx/facade/util/TypedId.java | 23 +- .../internal/connectors/CachedResource.java | 136 ++++++---- .../connectors/ConnectorsConnection.java | 6 +- .../connectors/ConnectorsDriverSupport.java | 2 +- .../connectors/GenericSDMXClientResource.java | 2 +- .../connectors/CachedResourceTest.java | 256 ++++++++++++++++++ ...est.java => ConnectorsConnectionTest.java} | 2 +- ...java => PortableTimeSeriesCursorTest.java} | 3 +- 9 files changed, 380 insertions(+), 65 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java rename sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/{SdmxConnectionAdapterTest.java => ConnectorsConnectionTest.java} (96%) rename sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/{DataCursorAdapterTest.java => PortableTimeSeriesCursorTest.java} (96%) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index d576edc7..22b22b73 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -69,6 +69,17 @@ public SdmxConnection asConnection() { seriesKeysOnlySupported); } + @Nonnull + public DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException { + Objects.requireNonNull(flowRef); + Objects.requireNonNull(query); + List col = data.get(flowRef); + if (col != null) { + return Series.asCursor(col, query.getKey()); + } + throw new IOException("Data not found"); + } + public static final class Builder { private final Map> data = new HashMap<>(); @@ -87,9 +98,7 @@ public Builder data(@Nonnull DataflowRef flowRef, @Nonnull Series series) { @Nonnull public Builder data(@Nonnull DataflowRef flowRef, @Nonnull List list) { - if (!list.isEmpty()) { - data.computeIfAbsent(flowRef, o -> new ArrayList<>()).addAll(list); - } + data.computeIfAbsent(flowRef, o -> new ArrayList<>()).addAll(list); return this; } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/TypedId.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/TypedId.java index 5c23e51a..b33de375 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/TypedId.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/TypedId.java @@ -16,20 +16,37 @@ */ package be.nbb.sdmx.facade.util; +import java.util.Objects; import javax.annotation.Nonnull; /** * * @author Philippe Charles + * @param */ -@lombok.Value(staticConstructor = "of") +@lombok.EqualsAndHashCode public final class TypedId { - @lombok.NonNull - String content; + @Nonnull + public static TypedId of(@Nonnull String content) { + Objects.requireNonNull(content); + return new TypedId<>(content); + } + + private final String content; + + private TypedId(String content) { + this.content = content; + } + + @Override + public String toString() { + return content; + } @Nonnull public TypedId with(@Nonnull Object o) { + Objects.requireNonNull(o); return new TypedId<>(content + o); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java index 2b911dbb..56b94f65 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java @@ -26,96 +26,128 @@ import be.nbb.sdmx.facade.util.TypedId; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; +import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import java.io.IOException; +import java.net.URL; import java.time.Clock; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentMap; /** * * @author Philippe Charles */ -final class CachedResource implements ConnectorsConnection.Resource { +final class CachedResource extends GenericSDMXClientResource { + + static CachedResource of(GenericSDMXClient client, URL endpoint, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { + return new CachedResource(client, generateBase(endpoint.getHost(), languages), cache, clock, ttlInMillis); + } - private final ConnectorsConnection.Resource delegate; private final TtlCache cache; - private final TypedId> dataflowsKey; - private final TypedId dataflowKey; - private final TypedId dataflowStructureKey; - private final TypedId dataKey; + private final TypedId> idOfFlows; + private final TypedId idOfFlow; + private final TypedId idOfStruct; + private final TypedId idOfKeysOnly; - CachedResource(ConnectorsConnection.Resource delegate, String host, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { - this.delegate = delegate; + CachedResource(GenericSDMXClient client, String base, ConcurrentMap cache, Clock clock, long ttlInMillis) { + super(client); this.cache = TtlCache.of(cache, clock, ttlInMillis); - String base = host + languages.toString(); - this.dataflowsKey = TypedId.of("flows://" + base); - this.dataflowKey = TypedId.of("flow://" + base + "/"); - this.dataflowStructureKey = TypedId.of("struct://" + base + "/"); - this.dataKey = TypedId.of("data://" + base + "/"); + this.idOfFlows = TypedId.of("flows://" + base); + this.idOfFlow = TypedId.of("flow://" + base); + this.idOfStruct = TypedId.of("struct://" + base); + this.idOfKeysOnly = TypedId.of("keys://" + base); } @Override public Map loadDataFlowsById() throws IOException { - Map result = cache.get(dataflowsKey); - if (result == null) { - result = delegate.loadDataFlowsById(); - cache.put(dataflowsKey, result); - } - return result; + return loadDataFlowsWithCache(); } @Override public Dataflow loadDataflow(DataflowRef flowRef) throws IOException { - // check if dataflow has been already loaded by #loadDataFlowsById - Map dataFlows = cache.get(dataflowsKey); - if (dataFlows != null) { - Dataflow tmp = dataFlows.get(flowRef.getId()); - if (tmp != null) { - return tmp; - } - } + Objects.requireNonNull(flowRef); - TypedId id = dataflowKey.with(flowRef); - Dataflow result = cache.get(id); + Dataflow result = peekDataflowFromCache(flowRef); + return result != null ? result : loadDataflowWithCache(flowRef); + } + + @Override + public DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { + return loadDataStructureWithCache(flowRef); + } + + @Override + public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + return serieskeysonly + ? loadKeysOnlyWithCache(flowRef, key).getData(flowRef, DataQuery.of(key, true)) + : super.loadData(flowRef, key, serieskeysonly); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return super.isSeriesKeysOnlySupported(); + } + + private Map loadDataFlowsWithCache() throws IOException { + Map result = cache.get(idOfFlows); if (result == null) { - result = delegate.loadDataflow(flowRef); - cache.put(id, result); + result = super.loadDataFlowsById(); + cache.put(idOfFlows, result); } return result; } - @Override - public DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { - TypedId id = dataflowStructureKey.with(flowRef); + private DataFlowStructure loadDataStructureWithCache(DataflowRef flowRef) throws IOException { + TypedId id = idOfStruct.with(flowRef); DataFlowStructure result = cache.get(id); if (result == null) { - result = delegate.loadDataStructure(flowRef); + result = super.loadDataStructure(flowRef); cache.put(id, result); } return result; } - @Override - public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - if (!serieskeysonly) { - return delegate.loadData(flowRef, key, serieskeysonly); - } - TypedId id = dataKey.with(flowRef); + private SdmxRepository loadKeysOnlyWithCache(DataflowRef flowRef, Key key) throws IOException { + TypedId id = idOfKeysOnly.with(flowRef); SdmxRepository result = cache.get(id); - if (result == null || key.supersedes(Key.parse(result.getName()))) { - try (DataCursor cursor = delegate.loadData(flowRef, key, true)) { - result = SdmxRepository.builder() - .copyOf(flowRef, cursor) - .name(key.toString()) - .build(); - } + if (result == null || isBroaderRequest(key, result)) { + result = copyDataKeys(flowRef, key); cache.put(id, result); } - return result.asConnection().getData(flowRef, DataQuery.of(key, true)); + return result; } - @Override - public boolean isSeriesKeysOnlySupported() { - return delegate.isSeriesKeysOnlySupported(); + private Dataflow peekDataflowFromCache(DataflowRef flowRef) { + // check if dataflow has been already loaded by #loadDataFlowsById + Map dataFlows = cache.get(idOfFlows); + return dataFlows != null ? dataFlows.get(flowRef.getId()) : null; + } + + private Dataflow loadDataflowWithCache(DataflowRef flowRef) throws IOException { + TypedId id = idOfFlow.with(flowRef); + Dataflow result = cache.get(id); + if (result == null) { + result = super.loadDataflow(flowRef); + cache.put(id, result); + } + return result; + } + + private boolean isBroaderRequest(Key key, SdmxRepository repo) { + return key.supersedes(Key.parse(repo.getName())); + } + + private SdmxRepository copyDataKeys(DataflowRef flowRef, Key key) throws IOException { + try (DataCursor cursor = super.loadData(flowRef, key, true)) { + return SdmxRepository.builder() + .copyOf(flowRef, cursor) + .name(key.toString()) + .build(); + } + } + + private static String generateBase(String host, LanguagePriorityList languages) { + return host + languages.toString() + "/"; } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java index 9111b768..86eebf06 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java @@ -42,13 +42,13 @@ interface Resource { Map loadDataFlowsById() throws IOException; @Nonnull - it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) throws IOException; + it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException; + it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException; + DataCursor loadData(@Nonnull DataflowRef flowRef, @Nonnull Key key, boolean serieskeysonly) throws IOException; boolean isSeriesKeysOnlySupported(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java index e3267df0..cb8f7369 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java @@ -97,7 +97,7 @@ private ConnectorsConnection.Resource getResource(URI uri, Map info, Langu URL endpoint = new URL(uri.toString().substring(prefix.length())); GenericSDMXClient client = supplier.getClient(endpoint, info, languages); applyTimeouts(client, info); - return new CachedResource(new GenericSDMXClientResource(client), endpoint.getHost(), languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); + return CachedResource.of(client, endpoint, languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); } catch (MalformedURLException ex) { throw new IOException(ex); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java index f9a23884..49b54782 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java @@ -37,7 +37,7 @@ */ @lombok.AllArgsConstructor @lombok.extern.java.Log -final class GenericSDMXClientResource implements ConnectorsConnection.Resource { +class GenericSDMXClientResource implements ConnectorsConnection.Resource { private final GenericSDMXClient client; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java new file mode 100644 index 00000000..5c6786ef --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java @@ -0,0 +1,256 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.TypedId; +import it.bancaditalia.oss.sdmx.api.DSDIdentifier; +import it.bancaditalia.oss.sdmx.api.DataFlowStructure; +import it.bancaditalia.oss.sdmx.api.Dataflow; +import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; +import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import java.io.IOException; +import java.net.URL; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class CachedResourceTest { + + @Test + public void testLoadDataFlowsById() throws IOException { + CountingClient resource = new CountingClient(); + ConcurrentHashMap cache = new ConcurrentHashMap(); + FakeClock clock = new FakeClock(); + + CachedResource target = new CachedResource(resource, "", cache, clock, 100); + + target.loadDataFlowsById(); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(flowsId); + + target.loadDataFlowsById(); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(flowsId); + + clock.plus(100); + target.loadDataFlowsById(); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(flowsId); + + cache.clear(); + target.loadDataFlowsById(); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(flowsId); + } + + @Test + public void testLoadDataflow() throws IOException { + CountingClient resource = new CountingClient(); + ConcurrentHashMap cache = new ConcurrentHashMap(); + FakeClock clock = new FakeClock(); + + CachedResource target = new CachedResource(resource, "", cache, clock, 100); + + assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + + target.loadDataflow(sample); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(flowId); + + target.loadDataflow(sample); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(flowId); + + clock.plus(100); + target.loadDataflow(sample); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(flowId); + + cache.clear(); + target.loadDataflow(sample); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(flowId); + + cache.clear(); + target.loadDataFlowsById(); + target.loadDataflow(sample); + assertThat(resource.count).isEqualTo(4); + assertThat(cache).containsOnlyKeys(flowsId); + } + + @Test + public void testLoadDataStructure() throws IOException { + CountingClient resource = new CountingClient(); + ConcurrentHashMap cache = new ConcurrentHashMap(); + FakeClock clock = new FakeClock(); + + CachedResource target = new CachedResource(resource, "", cache, clock, 100); + + assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + + target.loadDataStructure(sample); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(flowId, structId); + + target.loadDataStructure(sample); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(flowId, structId); + + clock.plus(100); + target.loadDataStructure(sample); + assertThat(resource.count).isEqualTo(4); + assertThat(cache).containsOnlyKeys(flowId, structId); + + cache.clear(); + target.loadDataStructure(sample); + assertThat(resource.count).isEqualTo(6); + assertThat(cache).containsOnlyKeys(flowId, structId); + } + + @Test + public void testLoadData() throws IOException { + CountingClient resource = new CountingClient(); + ConcurrentHashMap cache = new ConcurrentHashMap(); + FakeClock clock = new FakeClock(); + + CachedResource target = new CachedResource(resource, "", cache, clock, 100); + + assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + + target.loadData(sample, Key.ALL, true); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + + target.loadData(sample, Key.ALL, true); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + + clock.plus(100); + target.loadData(sample, Key.ALL, true); + assertThat(resource.count).isEqualTo(6); + assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + + cache.clear(); + target.loadData(sample, Key.ALL, true); + assertThat(resource.count).isEqualTo(9); + assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + } + + private final DataflowRef sample = DataflowRef.parse("sample"); + private final TypedId flowsId = TypedId.of("flows://"); + private final TypedId flowId = TypedId.of("flow://all,sample,latest"); + private final TypedId structId = TypedId.of("struct://all,sample,latest"); + private final TypedId keysId = TypedId.of("keys://all,sample,latest"); + + private static final class FakeClock extends Clock { + + private Instant current = Instant.now(); + + void plus(long durationInMillis) { + current = current.plus(100, ChronoUnit.MILLIS); + } + + @Override + public ZoneId getZone() { + return ZoneId.systemDefault(); + } + + @Override + public Clock withZone(ZoneId zone) { + return this; + } + + @Override + public Instant instant() { + return current; + } + } + + private static final class CountingClient implements GenericSDMXClient { + + int count = 0; + + @Override + public Map getDataflows() throws SdmxException { + count++; + return Collections.singletonMap("sample", new Dataflow()); + } + + @Override + public Dataflow getDataflow(String string, String string1, String string2) throws SdmxException { + count++; + return new Dataflow(); + } + + @Override + public DataFlowStructure getDataFlowStructure(DSDIdentifier dsdi, boolean bln) throws SdmxException { + count++; + return new DataFlowStructure(); + } + + @Override + public Map getCodes(String string, String string1, String string2) throws SdmxException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public List getTimeSeries(Dataflow dtflw, DataFlowStructure dfs, String string, String string1, String string2, boolean bln, String string3, boolean bln1) throws SdmxException { + count++; + return Collections.emptyList(); + } + + @Override + public boolean needsCredentials() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public void setCredentials(String string, String string1) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public URL getEndpoint() throws SdmxException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public void setEndpoint(URL url) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String buildDataURL(Dataflow dtflw, String string, String string1, String string2, boolean bln, String string3, boolean bln1) throws SdmxException { + throw new UnsupportedOperationException("Not supported yet."); + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java index d397f54f..879de953 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/SdmxConnectionAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java @@ -24,7 +24,7 @@ * * @author Philippe Charles */ -public class SdmxConnectionAdapterTest { +public class ConnectorsConnectionTest { @Test public void testCompliance() { diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java similarity index 96% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index 499133fb..e23c703d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/DataCursorAdapterTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -32,12 +32,13 @@ import org.junit.BeforeClass; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThat; /** * * @author Philippe Charles */ -public class DataCursorAdapterTest { +public class PortableTimeSeriesCursorTest { static List DATA; From 85f8e18ddb74a22f01ae61b4d46e07083682c5d5 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 20 Oct 2017 12:12:31 +0200 Subject: [PATCH 15/68] Code cleanup. --- .../internal/connectors/TestResource.java | 56 ++++---- .../file/DataStructureDecoderTest.java | 11 +- .../java/internal/file/DataTypeProbeTest.java | 11 +- .../internal/file/FileSdmxConnectionTest.java | 8 +- .../nbb/sdmx/facade/samples/ByteSource.java | 10 +- .../nbb/sdmx/facade/samples/SdmxSource.java | 11 +- .../java/be/nbb/sdmx/facade/repo/Obs.java | 4 + .../be/nbb/sdmx/facade/util/DataFactory.java | 4 - .../facade/xml/stream/SdmxXmlStreams.java | 9 +- .../internal/util/StandardDataFactory.java | 11 -- .../XMLStreamCompactDataCursorTest.java | 8 +- .../XMLStreamGenericDataCursorTest.java | 12 +- .../connectors/PortableTimeSeriesCursor.java | 6 +- .../connectors/drivers/InseeDriver.java | 29 ++-- .../connectors/drivers/Sdmx21Driver.java | 29 ++-- .../util/drivers/InseeDataFactory.java | 6 - .../connectors/ConnectorsResource.java | 125 +++++++++++------- .../internal/connectors/FacadeResource.java | 18 +-- 18 files changed, 195 insertions(+), 173 deletions(-) diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 19f08470..2d9972a4 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -23,8 +23,10 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; +import it.bancaditalia.oss.sdmx.client.Parser; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; import java.io.IOException; @@ -34,7 +36,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; /** @@ -48,27 +49,23 @@ public static final SdmxRepository nbb() { try { LanguagePriorityList l = LanguagePriorityList.parse("en"); SdmxRepository.Builder result = SdmxRepository.builder(); - Map dataStructures; - try (InputStreamReader r = SdmxSource.NBB_DATA_STRUCTURE.openReader()) { - dataStructures = toDataStructures(new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser().parse(r, l)); - result.dataStructures(dataStructures.values()); - } + Map dataStructures = toDataStructures(parse(SdmxSource.NBB_DATA_STRUCTURE, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser())); + result.dataStructures(dataStructures.values()); DataflowRef flowRef = DataflowRef.of("NBB", "TEST_DATASET", null); - try (InputStreamReader r = SdmxSource.NBB_DATAFLOWS.openReader()) { - result.dataflows(new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser().parse(r, l).stream() - .map(TestResource::toDataFlow) - .filter(o -> o.getFlowRef().equals(flowRef)) - .collect(Collectors.toList())); - } - DataStructure dfs = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); - try (DataCursor cursor = SdmxXmlStreams.genericData20(dfs).get(XMLInputFactory.newInstance(), SdmxSource.NBB_DATA.openReader())) { + result.dataflows(parse(SdmxSource.NBB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser()) + .stream() + .map(TestResource::toDataFlow) + .filter(o -> o.getFlowRef().equals(flowRef)) + .collect(Collectors.toList())); + DataStructure dsd = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); + try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).get(SdmxSource.XIF, SdmxSource.NBB_DATA.openReader())) { result.copyOf(flowRef, cursor); } return result .name("NBB") .seriesKeysOnlySupported(false) .build(); - } catch (IOException | XMLStreamException | SdmxException ex) { + } catch (IOException ex) { throw new RuntimeException(ex); } } @@ -78,27 +75,22 @@ public static final SdmxRepository ecb() { try { LanguagePriorityList l = LanguagePriorityList.parse("en"); SdmxRepository.Builder result = SdmxRepository.builder(); - Map dataStructures; - try (InputStreamReader r = SdmxSource.ECB_DATA_STRUCTURE.openReader()) { - dataStructures = toDataStructures(new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser().parse(r, l)); - result.dataStructures(dataStructures.values()); - } + Map dataStructures = toDataStructures(parse(SdmxSource.ECB_DATA_STRUCTURE, l, new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser())); + result.dataStructures(dataStructures.values()); DataflowRef flowRef = DataflowRef.of("ECB", "AME", "1.0"); - try (InputStreamReader r = SdmxSource.ECB_DATAFLOWS.openReader()) { - result.dataflows(new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser().parse(r, l).stream() - .map(Util::toDataflow) - .filter(o -> o.getFlowRef().equals(flowRef)) - .collect(Collectors.toList())); - } + result.dataflows(parse(SdmxSource.ECB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()).stream() + .map(Util::toDataflow) + .filter(o -> o.getFlowRef().equals(flowRef)) + .collect(Collectors.toList())); DataStructure dfs = dataStructures.get(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); - try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).get(XMLInputFactory.newInstance(), SdmxSource.ECB_DATA.openReader())) { + try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).get(SdmxSource.XIF, SdmxSource.ECB_DATA.openReader())) { result.copyOf(flowRef, cursor); } return result .name("ECB") .seriesKeysOnlySupported(true) .build(); - } catch (IOException | XMLStreamException | SdmxException ex) { + } catch (IOException ex) { throw new RuntimeException(ex); } } @@ -115,4 +107,12 @@ private static Dataflow toDataFlow(DataFlowStructure input) { input.getName() ); } + + private static T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { + try (InputStreamReader r = xml.openReader()) { + return parser.parse(r, l); + } catch (XMLStreamException | SdmxException ex) { + throw new IOException(ex); + } + } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index dd156583..711e1828 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -19,7 +19,6 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import java.io.IOException; -import javax.xml.stream.XMLInputFactory; import org.junit.Test; import be.nbb.sdmx.facade.samples.SdmxSource; import static org.assertj.core.api.Assertions.assertThat; @@ -32,8 +31,6 @@ */ public class DataStructureDecoderTest { - private final XMLInputFactory factory = XMLInputFactory.newInstance(); - @Test public void testDecodeGeneric20() throws IOException { DataStructure ds = DataStructure.builder() @@ -48,7 +45,7 @@ public void testDecodeGeneric20() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC20, factory, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC20, SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -67,7 +64,7 @@ public void testDecodeCompact20() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT20, factory, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT20, SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -88,7 +85,7 @@ public void testDecodeGeneric21() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC21, factory, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC21, SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -109,7 +106,7 @@ public void testDecodeCompact21() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT21, factory, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT21, SdmxSource.XIF, stream)).isEqualTo(ds); } } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java index e14033a7..09450b2c 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java @@ -20,7 +20,6 @@ import java.io.IOException; import org.junit.Test; import java.io.Reader; -import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.assertThat; /** @@ -29,33 +28,31 @@ */ public class DataTypeProbeTest { - private final XMLInputFactory factory = XMLInputFactory.newInstance(); - @Test public void testDecodeGeneric20() throws IOException { try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataTypeProbe.probeDataType(factory, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC20); + assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC20); } } @Test public void testDecodeCompact20() throws IOException { try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataTypeProbe.probeDataType(factory, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT20); + assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT20); } } @Test public void testDecodeGeneric21() throws IOException { try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataTypeProbe.probeDataType(factory, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC21); + assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC21); } } @Test public void testDecodeCompact21() throws IOException { try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataTypeProbe.probeDataType(factory, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT21); + assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT21); } } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index e3e98be0..385ed19e 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -26,7 +26,6 @@ import be.nbb.sdmx.facade.tck.ConnectionAssert; import java.io.File; import java.io.IOException; -import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; import org.junit.Rule; @@ -40,8 +39,7 @@ public class FileSdmxConnectionTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - private final XMLInputFactory factory = XMLInputFactory.newInstance(); - private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(factory); + private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(SdmxSource.XIF); @Test public void testCompactData21() throws IOException { @@ -50,7 +48,7 @@ public void testCompactData21() throws IOException { SdmxFile file = new SdmxFile(compact21, null); - FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, factory, decoder); + FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); assertThat(conn.getDataflows()).hasSize(1); assertThat(conn.getDataStructure(file.getDataflowRef()).getDimensions()).hasSize(7); @@ -78,6 +76,6 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(file, LanguagePriorityList.ANY, factory, decoder), file.getDataflowRef()); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), file.getDataflowRef()); } } diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java index 816987d2..78a230ea 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java @@ -26,6 +26,7 @@ import java.nio.file.StandardCopyOption; import java.util.Objects; import javax.annotation.Nonnull; +import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; @@ -40,8 +41,13 @@ public interface ByteSource { InputStream openStream() throws IOException; @Nonnull - default XMLStreamReader openXmlStream() throws XMLStreamException, IOException { - return XMLInputFactory.newInstance().createXMLStreamReader(openReader()); + default XMLStreamReader openXmlStream(@Nonnull XMLInputFactory f) throws XMLStreamException, IOException { + return f.createXMLStreamReader(openReader()); + } + + @Nonnull + default XMLEventReader openXmlEvent(@Nonnull XMLInputFactory f) throws XMLStreamException, IOException { + return f.createXMLEventReader(openReader()); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java index 2b57c1b4..7141403a 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java @@ -16,15 +16,14 @@ */ package be.nbb.sdmx.facade.samples; +import javax.xml.stream.XMLInputFactory; + /** * * @author Philippe Charles */ -public final class SdmxSource { - - private SdmxSource() { - // static class - } +@lombok.experimental.UtilityClass +public class SdmxSource { private static ByteSource of(String name) { return ByteSource.of(SdmxSource.class, name); @@ -42,4 +41,6 @@ private static ByteSource of(String name) { public static final ByteSource OTHER_COMPACT20 = of("other/sdmx-compact20-sample.xml"); public static final ByteSource OTHER_GENERIC21 = of("other/sdmx-GenericData21.xml"); public static final ByteSource OTHER_COMPACT21 = of("other/sdmx-compactData21.xml"); + + public static final XMLInputFactory XIF = XMLInputFactory.newInstance(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java index 93ce7737..d5c7aa4b 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java @@ -17,6 +17,7 @@ package be.nbb.sdmx.facade.repo; import java.time.LocalDateTime; +import javax.annotation.Nullable; /** * @@ -25,6 +26,9 @@ @lombok.Value(staticConstructor = "of") public class Obs { + @Nullable LocalDateTime period; + + @Nullable Double value; } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java index 7428b03f..02456ad6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java @@ -17,7 +17,6 @@ package be.nbb.sdmx.facade.util; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Key; import internal.util.StandardDataFactory; import javax.annotation.Nonnull; @@ -27,9 +26,6 @@ */ public interface DataFactory { - @Nonnull - Key.Builder getKeyBuilder(@Nonnull DataStructure dsd); - @Nonnull FreqParser getFreqParser(@Nonnull DataStructure dsd); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index 4742f4aa..72fbd26c 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -19,6 +19,7 @@ import be.nbb.sdmx.facade.util.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import java.io.IOException; import java.util.List; @@ -41,7 +42,7 @@ public XMLStream compactData20(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> new XMLStreamCompactDataCursor(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); + return o -> new XMLStreamCompactDataCursor(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); } @Nonnull @@ -51,7 +52,7 @@ public XMLStream compactData21(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> new XMLStreamCompactDataCursor(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + return o -> new XMLStreamCompactDataCursor(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull @@ -61,7 +62,7 @@ public XMLStream genericData20(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> XMLStreamGenericDataCursor.sdmx20(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return o -> XMLStreamGenericDataCursor.sdmx20(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull @@ -71,7 +72,7 @@ public XMLStream genericData21(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> XMLStreamGenericDataCursor.sdmx21(o, df.getKeyBuilder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return o -> XMLStreamGenericDataCursor.sdmx21(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java index 3e68744b..047c9ac6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java @@ -17,7 +17,6 @@ package internal.util; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.util.DataFactory; import be.nbb.sdmx.facade.util.FreqParser; import be.nbb.sdmx.facade.util.ObsParser; @@ -29,11 +28,6 @@ public enum StandardDataFactory implements DataFactory { SDMX20 { - @Override - public Key.Builder getKeyBuilder(DataStructure o) { - return Key.builder(o); - } - @Override public FreqParser getFreqParser(DataStructure o) { return FreqParser.sdmx20(); @@ -44,11 +38,6 @@ public ObsParser getObsParser(DataStructure o) { return ObsParser.standard(); } }, SDMX21 { - @Override - public Key.Builder getKeyBuilder(DataStructure o) { - return Key.builder(o); - } - @Override public FreqParser getFreqParser(DataStructure o) { return FreqParser.sdmx21(o); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index 9ea9b74b..a1d92a7f 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -39,9 +39,9 @@ public void testCompactData20() throws Exception { ByteSource xml = SdmxSource.OTHER_COMPACT20; Key.Builder builder = Key.builder("FREQ", "COLLECTION", "VIS_CTY", "JD_TYPE", "JD_CATEGORY"); - DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")); + DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")); - try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { + try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -79,9 +79,9 @@ public void testCompactData21() throws Exception { ByteSource xml = SdmxSource.OTHER_COMPACT21; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")); + DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")); - try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { + try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index cb1ea758..50b34a00 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -39,9 +39,9 @@ public void testGenericData20() throws Exception { ByteSource xml = SdmxSource.NBB_DATA; Key.Builder builder = Key.builder("SUBJECT", "LOCATION", "FREQUENCY"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20())); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20())); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx20())) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20())) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -83,9 +83,9 @@ public void testGenericData21() throws Exception { ByteSource xml = SdmxSource.OTHER_GENERIC21; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -119,9 +119,9 @@ public void testGenericData21Bis() throws Exception { ByteSource xml = SdmxSource.ECB_DATA; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))); + DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index 39549691..880e95ba 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -136,13 +136,13 @@ private void checkObsState() throws IOException, IllegalStateException { // private static Frequency getFrequency(PortableTimeSeries input) { + if (input.getFrequency() != null) { + return FreqUtil.parseByFreq(input.getFrequency()); + } String value = input.getAttribute(TIME_FORMAT_CONCEPT); if (value != null) { return FreqUtil.parseByTimeFormat(value); } - if (input.getFrequency() != null) { - return FreqUtil.parseByFreq(input.getFrequency()); - } return Frequency.UNDEFINED; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 22a0c574..ced30581 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -17,7 +17,6 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; @@ -31,12 +30,10 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; import java.io.IOException; -import java.io.Reader; import java.net.URL; import java.util.Collection; import java.util.List; import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.HasDataCursor; @@ -44,6 +41,7 @@ import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; import internal.util.drivers.InseeDataFactory; +import it.bancaditalia.oss.sdmx.client.Parser; /** * @@ -79,10 +77,8 @@ private InseeClient(URL endpoint, LanguagePriorityList langs, XMLInputFactory xm @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { - String query = buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false); // FIXME: avoid in-memory copy - List data = runQuery((r, l) -> parse(r, Util.toDataStructure(dsd)), query, SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); - return Series.asCursor(data, resource); + return Series.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -90,12 +86,21 @@ public boolean isSeriesKeysOnlySupported() { return true; } - private List parse(Reader xmlReader, DataStructure dsd) throws XMLStreamException, SdmxException { - try { - return Series.copyOf(SdmxXmlStreams.compactData21(dsd, dataFactory).get(xmlFactory, xmlReader)); - } catch (IOException ex) { - throw new SdmxIOException("Cannot parse compact data 21", ex); - } + private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { + return runQuery( + getCompactData21Parser(dsd), + buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false), + SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); + } + + private Parser> getCompactData21Parser(DataFlowStructure dsd) { + return (r, l) -> { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(xmlFactory, r)) { + return Series.copyOf(cursor); + } catch (IOException ex) { + throw new SdmxIOException("Cannot parse compact data 21", ex); + } + }; } } // diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 6414113c..079e71e5 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -17,7 +17,6 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; @@ -31,20 +30,19 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; import java.io.IOException; -import java.io.Reader; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; +import it.bancaditalia.oss.sdmx.client.Parser; /** * @@ -185,10 +183,8 @@ private Sdmx21Client(URL endpoint, Sdmx21Config config, LanguagePriorityList lan @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { - String query = buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false); // FIXME: avoid in-memory copy - List data = runQuery((r, l) -> parse(r, Util.toDataStructure(dsd)), query, SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); - return Series.asCursor(data, resource); + return Series.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -196,12 +192,21 @@ public boolean isSeriesKeysOnlySupported() { return config.isSeriesKeysOnlySupported(); } - private List parse(Reader xmlReader, DataStructure dsd) throws XMLStreamException, SdmxException { - try { - return Series.copyOf(SdmxXmlStreams.compactData21(dsd).get(factory, xmlReader)); - } catch (IOException ex) { - throw new SdmxIOException("Cannot parse compact data 21", ex); - } + private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { + return runQuery( + getCompactData21Parser(dsd), + buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false), + SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); + } + + private Parser> getCompactData21Parser(DataFlowStructure dsd) { + return (r, l) -> { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(factory, r)) { + return Series.copyOf(cursor); + } catch (IOException ex) { + throw new SdmxIOException("Cannot parse compact data 21", ex); + } + }; } } // diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java index 1585a791..d951ded5 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java @@ -22,7 +22,6 @@ import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; import static be.nbb.sdmx.facade.Frequency.MONTHLY; import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.util.FreqParser; import be.nbb.sdmx.facade.util.ObsParser; import be.nbb.sdmx.facade.util.SafeParser; @@ -37,11 +36,6 @@ */ public final class InseeDataFactory implements DataFactory { - @Override - public Key.Builder getKeyBuilder(DataStructure dsd) { - return Key.builder(dsd); - } - @Override public FreqParser getFreqParser(DataStructure dsd) { return FreqParser.of(FreqUtil.extractorByIndex(dsd), InseeDataFactory::parseInseeFreq); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index d453cb6a..e417e50a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -16,26 +16,26 @@ */ package internal.connectors; -import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.repo.Series; import be.nbb.sdmx.facade.util.ObsParser; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.api.Dimension; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; +import it.bancaditalia.oss.sdmx.client.Parser; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.parser.v20.GenericDataParser; import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; import java.io.IOException; import java.io.InputStreamReader; import java.time.LocalDateTime; -import java.util.ArrayList; +import java.time.YearMonth; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -92,11 +92,7 @@ DataflowRef firstOf(List flows) { } List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser().parse(r, l); - } catch (XMLStreamException | SdmxException ex) { - throw new IOException(ex); - } + return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser()); } List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { @@ -106,52 +102,56 @@ List flow20(ByteSource xml, LanguagePriorityList l) throws IOException } List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return new GenericDataParser(dsd, null, true).parse(r, l).getData(); - } catch (XMLStreamException | SdmxException ex) { - throw new IOException(ex); - } + // No connectors impl + return FacadeResource.data20(XMLInputFactory.newFactory(), xml, Util.toDataStructure(dsd)) + .stream() + .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) + .collect(Collectors.toList()); } List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser().parse(r, l); - } catch (XMLStreamException | SdmxException ex) { - throw new IOException(ex); - } + return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser()); } List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser().parse(r, l); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()); } List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { - // FIXME: no connectors impl yet - try (DataCursor cursor = SdmxXmlStreams.genericData21(Util.toDataStructure(dsd)).get(XMLInputFactory.newFactory(), xml.openReader())) { - List dims = dsd.getDimensions(); - List result = new ArrayList<>(); - while (cursor.nextSeries()) { - PortableTimeSeries series = new PortableTimeSeries(); - series.setFrequency("A"); - cursor.getSeriesAttributes().forEach(series::addAttribute); - Key key = cursor.getSeriesKey(); - for (int i = 0; i < key.size(); i++) { - series.addDimension(dims.get(i).getId(), key.get(i)); - } - while (cursor.nextObs()) { - LocalDateTime period = cursor.getObsPeriod(); - if (period != null) { - Double value = cursor.getObsValue(); - series.addObservation(value != null ? value.toString() : "", String.valueOf(period.getYear()), null); - } - } - result.add(series); - } - return result; + // No connectors impl + return FacadeResource.data21(XMLInputFactory.newFactory(), xml, Util.toDataStructure(dsd)) + .stream() + .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) + .collect(Collectors.toList()); + } + + PortableTimeSeries toPortableTimeSeries(Series o, List dims) { + PortableTimeSeries result = new PortableTimeSeries(); + result.setFrequency(String.valueOf(formatByStandardFreq(o.getFrequency()))); + o.getMeta().forEach(result::addAttribute); + Key key = o.getKey(); + for (int i = 0; i < key.size(); i++) { + result.addDimension(dims.get(i).getId(), key.get(i)); + } + o.getObs().forEach(x -> result.addObservation(valueToString(x.getValue()), periodToString(o.getFrequency(), x.getPeriod()), null)); + return result; + } + + String valueToString(Double o) { + return o != null ? o.toString() : ""; + } + + String periodToString(Frequency f, LocalDateTime o) { + if (o == null) { + return ""; + } + switch (f) { + case ANNUAL: + return String.valueOf(o.getYear()); + case MONTHLY: + return YearMonth.from(o).toString(); + default: + throw new RuntimeException("Not implemented yet"); } } @@ -164,4 +164,37 @@ Dataflow asDataflow(DataFlowStructure o) { result.setVersion(o.getVersion()); return result; } + + T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { + try (InputStreamReader r = xml.openReader()) { + return parser.parse(r, l); + } catch (XMLStreamException | SdmxException ex) { + throw new IOException(ex); + } + } + + private char formatByStandardFreq(Frequency code) { + switch (code) { + case ANNUAL: + return 'A'; + case HALF_YEARLY: + return 'S'; + case QUARTERLY: + return 'Q'; + case MONTHLY: + return 'M'; + case WEEKLY: + return 'W'; + case DAILY: + return 'D'; + case HOURLY: + return 'H'; + case DAILY_BUSINESS: + return 'B'; + case MINUTELY: + return 'N'; + default: + return '?'; + } + } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index 7c4fe463..6144fabe 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.stream.Collectors; import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; /** * @@ -90,7 +89,7 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit .collect(Collectors.toList()); } - private List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { + List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData20(dsd).get(f, xml.openReader())) { return Series.copyOf(c); } @@ -103,17 +102,14 @@ private List struct21(XMLInputFactory f, ByteSource xml, Language } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - // FIXME: no facade impl yet - return new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser().parse(r, Util.fromLanguages(l)).stream() - .map(Util::toDataflow) - .collect(Collectors.toList()); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + // FIXME: no facade impl yet + return ConnectorsResource.parse(xml, Util.fromLanguages(l), new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()) + .stream() + .map(Util::toDataflow) + .collect(Collectors.toList()); } - private List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { + List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData21(dsd).get(f, xml.openReader())) { return Series.copyOf(c); } From 6818b3f479bdbe004536731a5460bdb327ca9781 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 24 Oct 2017 12:26:41 +0200 Subject: [PATCH 16/68] Updated connectors dependency. --- .../internal/connectors/TestResource.java | 14 +- sdmx-facade/sdmx-facade-web/pom.xml | 2 +- .../internal/connectors/CachedResource.java | 4 +- .../connectors/ConnectorsDriverSupport.java | 7 +- .../connectors/GenericSDMXClientSupplier.java | 7 +- .../connectors/drivers/InseeDriver.java | 14 +- .../connectors/drivers/Sdmx20Driver.java | 4 +- .../connectors/drivers/Sdmx21Driver.java | 21 +- .../internal/connectors/drivers/WbDriver.java | 43 --- .../util/xml/AbstractXMLStreamReader.java | 196 +++++++++++++ .../util/xml/XMLEventStreamReader.java | 276 ++++++++++++++++++ .../connectors/CachedResourceTest.java | 6 +- .../connectors/ConnectorsResource.java | 14 +- 13 files changed, 527 insertions(+), 81 deletions(-) delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 2d9972a4..043e27de 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -30,12 +30,12 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; import java.io.IOException; -import java.io.InputStreamReader; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLStreamException; /** @@ -109,10 +109,20 @@ private static Dataflow toDataFlow(DataFlowStructure input) { } private static T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { - try (InputStreamReader r = xml.openReader()) { + XMLEventReader r = null; + try { + r = xml.openXmlEvent(SdmxSource.XIF); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); + } finally { + if (r != null) { + try { + r.close(); + } catch (XMLStreamException ex) { + throw new RuntimeException(ex); + } + } } } } diff --git a/sdmx-facade/sdmx-facade-web/pom.xml b/sdmx-facade/sdmx-facade-web/pom.xml index d0319c1b..592d8d55 100644 --- a/sdmx-facade/sdmx-facade-web/pom.xml +++ b/sdmx-facade/sdmx-facade-web/pom.xml @@ -13,7 +13,7 @@ - e401cd23be + 346366c9d7 RELEASE802 diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java index 56b94f65..6d8807e9 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java @@ -28,7 +28,7 @@ import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.time.Clock; import java.util.Map; import java.util.Objects; @@ -40,7 +40,7 @@ */ final class CachedResource extends GenericSDMXClientResource { - static CachedResource of(GenericSDMXClient client, URL endpoint, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { + static CachedResource of(GenericSDMXClient client, URI endpoint, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { return new CachedResource(client, generateBase(endpoint.getHost(), languages), cache, clock, ttlInMillis); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java index cb8f7369..8f92df6b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java @@ -26,9 +26,8 @@ import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URI; -import java.net.URL; +import java.net.URISyntaxException; import java.time.Clock; import java.util.Collection; import java.util.Collections; @@ -94,11 +93,11 @@ public static Collection entry(@Nonnull String name, @Nonnull private ConnectorsConnection.Resource getResource(URI uri, Map info, LanguagePriorityList languages) throws IOException { try { - URL endpoint = new URL(uri.toString().substring(prefix.length())); + URI endpoint = new URI(uri.toString().substring(prefix.length())); GenericSDMXClient client = supplier.getClient(endpoint, info, languages); applyTimeouts(client, info); return CachedResource.of(client, endpoint, languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); - } catch (MalformedURLException ex) { + } catch (URISyntaxException ex) { throw new IOException(ex); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java index 8674da1a..fd44d316 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java @@ -19,8 +19,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.util.Map; import javax.annotation.Nonnull; @@ -31,11 +30,11 @@ public interface GenericSDMXClientSupplier { @Nonnull - GenericSDMXClient getClient(@Nonnull URL endpoint, @Nonnull Map info, @Nonnull LanguagePriorityList langs) throws MalformedURLException; + GenericSDMXClient getClient(@Nonnull URI endpoint, @Nonnull Map info, @Nonnull LanguagePriorityList langs); @Nonnull static GenericSDMXClientSupplier ofType(@Nonnull Class clazz) { - return (URL endpoint, Map info, LanguagePriorityList langs) -> { + return (URI endpoint, Map info, LanguagePriorityList langs) -> { try { RestSdmxClient result = clazz.newInstance(); result.setEndpoint(endpoint); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index ced30581..7e1a2c6e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -30,18 +30,18 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; import java.io.IOException; -import java.net.URL; import java.util.Collection; import java.util.List; -import javax.xml.stream.XMLInputFactory; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; +import internal.org.springframework.util.xml.XMLEventStreamReader; import internal.util.drivers.InseeDataFactory; import it.bancaditalia.oss.sdmx.client.Parser; +import java.net.URI; /** * @@ -52,10 +52,8 @@ public final class InseeDriver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:insee:"; - private final XMLInputFactory xml = XMLInputFactory.newInstance(); - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l, xml)); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l)); @Override public Collection getDefaultEntryPoints() { @@ -65,13 +63,11 @@ public Collection getDefaultEntryPoints() { // private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { - private final XMLInputFactory xmlFactory; private final InseeDataFactory dataFactory; - private InseeClient(URL endpoint, LanguagePriorityList langs, XMLInputFactory xmlFactory) { + private InseeClient(URI endpoint, LanguagePriorityList langs) { super("", endpoint, false, false, true); this.languages = Util.fromLanguages(langs); - this.xmlFactory = xmlFactory; this.dataFactory = new InseeDataFactory(); } @@ -95,7 +91,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(xmlFactory, r)) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { return Series.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java index fc27b856..e1f60b5e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java @@ -21,7 +21,6 @@ import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.RestSdmx20Client; -import java.net.URL; import java.util.Collections; import java.util.List; import java.util.Map; @@ -29,6 +28,7 @@ import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; +import java.net.URI; /** * @@ -49,7 +49,7 @@ public List getDefaultEntryPoints() { private static final class Sdmx20Client extends RestSdmx20Client { - private Sdmx20Client(URL endpoint, Map info, LanguagePriorityList langs) { + private Sdmx20Client(URI endpoint, Map info, LanguagePriorityList langs) { super("", endpoint, NEEDS_CREDENTIALS.get(info, false), null, "compact_v2"); this.languages = Util.fromLanguages(langs); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 079e71e5..100bf048 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -30,19 +30,19 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; import java.io.IOException; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.xml.stream.XMLInputFactory; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; +import internal.org.springframework.util.xml.XMLEventStreamReader; import it.bancaditalia.oss.sdmx.client.Parser; +import java.net.URI; /** * @@ -53,10 +53,8 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx21:"; - private final XMLInputFactory xml = XMLInputFactory.newInstance(); - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l, xml)); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l)); @Override public List getDefaultEntryPoints() { @@ -99,6 +97,13 @@ public List getDefaultEntryPoints() { .supportsCompression(true) .seriesKeysOnlySupported(true) .build()); + result.add(b.clear() + .name("WB") + .description("World Bank") + .endpoint("http://api.worldbank.org/v2/sdmx/rest") + .supportsCompression(true) +// .seriesKeysOnlySupported(true) + .build()); return result; } @@ -172,13 +177,11 @@ public SdmxWebEntryPoint build() { private final static class Sdmx21Client extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { private final Sdmx21Config config; - private final XMLInputFactory factory; - private Sdmx21Client(URL endpoint, Sdmx21Config config, LanguagePriorityList langs, XMLInputFactory factory) { + private Sdmx21Client(URI endpoint, Sdmx21Config config, LanguagePriorityList langs) { super("", endpoint, config.isNeedsCredentials(), config.isNeedsURLEncoding(), config.isSupportsCompression()); this.languages = Util.fromLanguages(langs); this.config = config; - this.factory = factory; } @Override @@ -201,7 +204,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(factory, r)) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(new XMLEventStreamReader(r))) { return Series.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java deleted file mode 100644 index 7cf760d7..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/WbDriver.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors.drivers; - -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; -import be.nbb.sdmx.facade.util.HasCache; -import it.bancaditalia.oss.sdmx.client.custom.WB; -import java.util.Collection; -import org.openide.util.lookup.ServiceProvider; -import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; - -/** - * - * @author Philippe Charles - */ -@ServiceProvider(service = SdmxWebDriver.class) -public final class WbDriver implements SdmxWebDriver, HasCache { - - private static final String PREFIX = "sdmx:wb:"; - - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, WB.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("WB", "World Bank", "sdmx:wb:http://api.worldbank.org"); - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java new file mode 100644 index 00000000..ba13ded4 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java @@ -0,0 +1,196 @@ +/* + * Copyright 2002-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package internal.org.springframework.util.xml; + +import javax.xml.namespace.QName; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Abstract base class for {@code XMLStreamReader}s. + * + * @author Arjen Poutsma + * @since 3.0 + */ +abstract class AbstractXMLStreamReader implements XMLStreamReader { + + @Override + public String getElementText() throws XMLStreamException { + if (getEventType() != XMLStreamConstants.START_ELEMENT) { + throw new XMLStreamException("parser must be on START_ELEMENT to read next text", getLocation()); + } + int eventType = next(); + StringBuilder builder = new StringBuilder(); + while (eventType != XMLStreamConstants.END_ELEMENT) { + if (eventType == XMLStreamConstants.CHARACTERS || eventType == XMLStreamConstants.CDATA + || eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.ENTITY_REFERENCE) { + builder.append(getText()); + } else if (eventType == XMLStreamConstants.PROCESSING_INSTRUCTION + || eventType == XMLStreamConstants.COMMENT) { + // skipping + } else if (eventType == XMLStreamConstants.END_DOCUMENT) { + throw new XMLStreamException("unexpected end of document when reading element text content", + getLocation()); + } else if (eventType == XMLStreamConstants.START_ELEMENT) { + throw new XMLStreamException("element text content may not contain START_ELEMENT", getLocation()); + } else { + throw new XMLStreamException("Unexpected event type " + eventType, getLocation()); + } + eventType = next(); + } + return builder.toString(); + } + + @Override + public String getAttributeLocalName(int index) { + return getAttributeName(index).getLocalPart(); + } + + @Override + public String getAttributeNamespace(int index) { + return getAttributeName(index).getNamespaceURI(); + } + + @Override + public String getAttributePrefix(int index) { + return getAttributeName(index).getPrefix(); + } + + @Override + public String getNamespaceURI() { + int eventType = getEventType(); + if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { + return getName().getNamespaceURI(); + } else { + throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); + } + } + + @Override + public String getNamespaceURI(String prefix) { +// Assert.notNull(prefix, "No prefix given"); + return getNamespaceContext().getNamespaceURI(prefix); + } + + @Override + public boolean hasText() { + int eventType = getEventType(); + return eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.CHARACTERS + || eventType == XMLStreamConstants.COMMENT || eventType == XMLStreamConstants.CDATA + || eventType == XMLStreamConstants.ENTITY_REFERENCE; + } + + @Override + public String getPrefix() { + int eventType = getEventType(); + if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { + return getName().getPrefix(); + } else { + throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); + } + } + + @Override + public boolean hasName() { + int eventType = getEventType(); + return eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT; + } + + @Override + public boolean isWhiteSpace() { + return getEventType() == XMLStreamConstants.SPACE; + } + + @Override + public boolean isStartElement() { + return getEventType() == XMLStreamConstants.START_ELEMENT; + } + + @Override + public boolean isEndElement() { + return getEventType() == XMLStreamConstants.END_ELEMENT; + } + + @Override + public boolean isCharacters() { + return getEventType() == XMLStreamConstants.CHARACTERS; + } + + @Override + public int nextTag() throws XMLStreamException { + int eventType = next(); + while (eventType == XMLStreamConstants.CHARACTERS && isWhiteSpace() + || eventType == XMLStreamConstants.CDATA && isWhiteSpace() || eventType == XMLStreamConstants.SPACE + || eventType == XMLStreamConstants.PROCESSING_INSTRUCTION || eventType == XMLStreamConstants.COMMENT) { + eventType = next(); + } + if (eventType != XMLStreamConstants.START_ELEMENT && eventType != XMLStreamConstants.END_ELEMENT) { + throw new XMLStreamException("expected start or end tag", getLocation()); + } + return eventType; + } + + @Override + public void require(int expectedType, String namespaceURI, String localName) throws XMLStreamException { + int eventType = getEventType(); + if (eventType != expectedType) { + throw new XMLStreamException("Expected [" + expectedType + "] but read [" + eventType + "]"); + } + } + + @Override + @javax.annotation.Nullable + public String getAttributeValue(@javax.annotation.Nullable String namespaceURI, String localName) { + for (int i = 0; i < getAttributeCount(); i++) { + QName name = getAttributeName(i); + if (name.getLocalPart().equals(localName) + && (namespaceURI == null || name.getNamespaceURI().equals(namespaceURI))) { + return getAttributeValue(i); + } + } + return null; + } + + @Override + public boolean hasNext() throws XMLStreamException { + return getEventType() != END_DOCUMENT; + } + + @Override + public String getLocalName() { + return getName().getLocalPart(); + } + + @Override + public char[] getTextCharacters() { + return getText().toCharArray(); + } + + @Override + public int getTextCharacters(int sourceStart, char[] target, int targetStart, int length) + throws XMLStreamException { + char[] source = getTextCharacters(); + length = Math.min(length, source.length); + System.arraycopy(source, sourceStart, target, targetStart, length); + return length; + } + + @Override + public int getTextLength() { + return getText().length(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java new file mode 100644 index 00000000..31ceb04d --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java @@ -0,0 +1,276 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package internal.org.springframework.util.xml; + +import java.util.Iterator; +import javax.xml.namespace.NamespaceContext; +import javax.xml.namespace.QName; +import javax.xml.stream.Location; +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.Attribute; +import javax.xml.stream.events.Comment; +import javax.xml.stream.events.Namespace; +import javax.xml.stream.events.ProcessingInstruction; +import javax.xml.stream.events.StartDocument; +import javax.xml.stream.events.XMLEvent; + +/** + * Implementation of the {@link javax.xml.stream.XMLStreamReader} interface that + * wraps a {@link XMLEventReader}. Useful because the StAX + * {@link javax.xml.stream.XMLInputFactory} allows one to create a event reader + * from a stream reader, but not vice-versa. + * + * @author Arjen Poutsma + * @since 3.0 + * @see StaxUtils#createEventStreamReader(javax.xml.stream.XMLEventReader) + */ +public final class XMLEventStreamReader extends AbstractXMLStreamReader { + + private XMLEvent event; + + private final XMLEventReader eventReader; + + public XMLEventStreamReader(XMLEventReader eventReader) throws XMLStreamException { + this.eventReader = eventReader; + this.event = eventReader.nextEvent(); + } + + @Override + public QName getName() { + if (this.event.isStartElement()) { + return this.event.asStartElement().getName(); + } else if (this.event.isEndElement()) { + return this.event.asEndElement().getName(); + } else { + throw new IllegalStateException(); + } + } + + @Override + public Location getLocation() { + return this.event.getLocation(); + } + + @Override + public int getEventType() { + return this.event.getEventType(); + } + + @Override + @javax.annotation.Nullable + public String getVersion() { + if (this.event.isStartDocument()) { + return ((StartDocument) this.event).getVersion(); + } else { + return null; + } + } + + @Override + public Object getProperty(String name) throws IllegalArgumentException { + return this.eventReader.getProperty(name); + } + + @Override + public boolean isStandalone() { + if (this.event.isStartDocument()) { + return ((StartDocument) event).isStandalone(); + } else { + throw new IllegalStateException(); + } + } + + @Override + public boolean standaloneSet() { + if (this.event.isStartDocument()) { + return ((StartDocument) this.event).standaloneSet(); + } else { + throw new IllegalStateException(); + } + } + + @Override + @javax.annotation.Nullable + public String getEncoding() { + return null; + } + + @Override + @javax.annotation.Nullable + public String getCharacterEncodingScheme() { + return null; + } + + @Override + public String getPITarget() { + if (this.event.isProcessingInstruction()) { + return ((ProcessingInstruction) this.event).getTarget(); + } else { + throw new IllegalStateException(); + } + } + + @Override + public String getPIData() { + if (this.event.isProcessingInstruction()) { + return ((ProcessingInstruction) this.event).getData(); + } else { + throw new IllegalStateException(); + } + } + + @Override + public int getTextStart() { + return 0; + } + + @Override + public String getText() { + if (this.event.isCharacters()) { + return event.asCharacters().getData(); + } else if (this.event.getEventType() == XMLEvent.COMMENT) { + return ((Comment) this.event).getText(); + } else { + throw new IllegalStateException(); + } + } + + @Override + @SuppressWarnings("rawtypes") + public int getAttributeCount() { + if (!this.event.isStartElement()) { + throw new IllegalStateException(); + } + Iterator attributes = this.event.asStartElement().getAttributes(); + return countIterator(attributes); + } + + @Override + public boolean isAttributeSpecified(int index) { + return getAttribute(index).isSpecified(); + } + + @Override + public QName getAttributeName(int index) { + return getAttribute(index).getName(); + } + + @Override + public String getAttributeType(int index) { + return getAttribute(index).getDTDType(); + } + + @Override + public String getAttributeValue(int index) { + return getAttribute(index).getValue(); + } + + @SuppressWarnings("rawtypes") + private Attribute getAttribute(int index) { + if (!this.event.isStartElement()) { + throw new IllegalStateException(); + } + int count = 0; + Iterator attributes = this.event.asStartElement().getAttributes(); + while (attributes.hasNext()) { + Attribute attribute = (Attribute) attributes.next(); + if (count == index) { + return attribute; + } else { + count++; + } + } + throw new IllegalArgumentException(); + } + + @Override + public NamespaceContext getNamespaceContext() { + if (this.event.isStartElement()) { + return this.event.asStartElement().getNamespaceContext(); + } else { + throw new IllegalStateException(); + } + } + + @Override + @SuppressWarnings("rawtypes") + public int getNamespaceCount() { + Iterator namespaces; + if (this.event.isStartElement()) { + namespaces = this.event.asStartElement().getNamespaces(); + } else if (this.event.isEndElement()) { + namespaces = this.event.asEndElement().getNamespaces(); + } else { + throw new IllegalStateException(); + } + return countIterator(namespaces); + } + + @Override + public String getNamespacePrefix(int index) { + return getNamespace(index).getPrefix(); + } + + @Override + public String getNamespaceURI(int index) { + return getNamespace(index).getNamespaceURI(); + } + + @SuppressWarnings("rawtypes") + private Namespace getNamespace(int index) { + Iterator namespaces; + if (this.event.isStartElement()) { + namespaces = this.event.asStartElement().getNamespaces(); + } else if (this.event.isEndElement()) { + namespaces = this.event.asEndElement().getNamespaces(); + } else { + throw new IllegalStateException(); + } + int count = 0; + while (namespaces.hasNext()) { + Namespace namespace = (Namespace) namespaces.next(); + if (count == index) { + return namespace; + } else { + count++; + } + } + throw new IllegalArgumentException(); + } + + @Override + public int next() throws XMLStreamException { + this.event = this.eventReader.nextEvent(); + return this.event.getEventType(); + } + + @Override + public void close() throws XMLStreamException { + this.eventReader.close(); + } + + @SuppressWarnings("rawtypes") + private static int countIterator(Iterator iterator) { + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + return count; + } + +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java index 5c6786ef..aa244093 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java @@ -26,7 +26,7 @@ import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; @@ -239,12 +239,12 @@ public void setCredentials(String string, String string1) { } @Override - public URL getEndpoint() throws SdmxException { + public URI getEndpoint() throws SdmxException { throw new UnsupportedOperationException("Not supported yet."); } @Override - public void setEndpoint(URL url) { + public void setEndpoint(URI url) { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index e417e50a..16cb851b 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -33,12 +33,12 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; import java.io.IOException; -import java.io.InputStreamReader; import java.time.LocalDateTime; import java.time.YearMonth; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; @@ -166,10 +166,20 @@ Dataflow asDataflow(DataFlowStructure o) { } T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { - try (InputStreamReader r = xml.openReader()) { + XMLEventReader r = null; + try { + r = xml.openXmlEvent(SdmxSource.XIF); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); + } finally { + if (r != null) { + try { + r.close(); + } catch (XMLStreamException ex) { + throw new RuntimeException(ex); + } + } } } From 3f056b9c502a5671845aef58c4dab532b979819a Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 25 Oct 2017 10:39:41 +0200 Subject: [PATCH 17/68] Improved Series#copyOf(). --- .../java/be/nbb/sdmx/facade/repo/Series.java | 14 ++++- .../be/nbb/sdmx/facade/repo/SeriesTest.java | 57 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java index cfe9620e..d510d34a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java @@ -55,12 +55,16 @@ public static List copyOf(@Nonnull DataCursor cursor) throws IOException if (!cursor.nextSeries()) { return Collections.emptyList(); } + + if (cursor instanceof SeriesCursor) { + return ((SeriesCursor) cursor).getRemainingItems(); + } + Series.Builder b = builder(); List result = new ArrayList<>(); - result.add(getSeries(b, cursor)); - while (cursor.nextSeries()) { + do { result.add(getSeries(b, cursor)); - } + } while (cursor.nextSeries()); return result; } @@ -161,6 +165,10 @@ public void close() throws IOException { closed = true; } + List getRemainingItems() { + return col.subList(i, col.size()); + } + private void checkState() throws IOException { if (closed) { throw new IOException("Cursor closed"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java new file mode 100644 index 00000000..8eba9f90 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.repo; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.util.NoOpCursor; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.List; +import javax.xml.stream.XMLStreamException; +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SeriesTest { + + @Test + public void testCopyOf() throws IOException, XMLStreamException { + try (DataCursor c = NoOpCursor.noOp()) { + assertThat(Series.copyOf(c)).isEmpty(); + } + + Series s1 = Series.builder().key(Key.of("hello")).frequency(Frequency.WEEKLY).obs(Obs.of(LocalDateTime.now(), 3.14)).build(); + try (DataCursor c = Series.asCursor(Collections.singletonList(s1), Key.ALL)) { + assertThat(Series.copyOf(c)).hasSize(1).element(0).isSameAs(s1); + } + + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); + try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + assertThat(Series.copyOf(o)).hasSize(120); + } + } +} From b5b2e94e3e1619bff71fff7ac11f64871c7f8743 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 26 Oct 2017 14:01:51 +0200 Subject: [PATCH 18/68] Moved Series & Obs to API. --- .../demetra/sdmx/web/SdmxWebProviderTest.java | 4 +- .../main/java/be/nbb/sdmx/facade}/Obs.java | 2 +- .../main/java/be/nbb/sdmx/facade/Series.java | 43 ++++ .../file/CachedFileSdmxConnection.java | 5 +- .../nbb/sdmx/facade/repo/SdmxRepository.java | 151 +++++++++++++- .../java/be/nbb/sdmx/facade/repo/Series.java | 193 ------------------ .../sdmx/facade/repo/SdmxRepositoryTest.java | 28 ++- .../be/nbb/sdmx/facade/repo/SeriesTest.java | 57 ------ .../connectors/drivers/InseeDriver.java | 7 +- .../connectors/drivers/Sdmx21Driver.java | 7 +- .../connectors/ConnectorsResource.java | 2 +- .../internal/connectors/FacadeResource.java | 6 +- .../PortableTimeSeriesCursorTest.java | 7 +- 13 files changed, 239 insertions(+), 273 deletions(-) rename sdmx-facade/{sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo => sdmx-facade-api/src/main/java/be/nbb/sdmx/facade}/Obs.java (96%) create mode 100644 sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java index a013593d..6427fad8 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java @@ -26,9 +26,9 @@ import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.repo.Obs; +import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Series; import ec.tss.TsMoniker; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.DataSource; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Obs.java similarity index 96% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Obs.java index d5c7aa4b..b8ce1fa8 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Obs.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Obs.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.repo; +package be.nbb.sdmx.facade; import java.time.LocalDateTime; import javax.annotation.Nullable; diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java new file mode 100644 index 00000000..6aaac08d --- /dev/null +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade; + +import java.util.List; +import java.util.Map; + +/** + * + * @author Philippe Charles + */ +@lombok.Value +@lombok.Builder(builderClassName = "Builder") +public class Series { + + @lombok.NonNull + Key key; + + @lombok.NonNull + Frequency frequency; + + @lombok.NonNull + @lombok.Singular(value = "obs") + List obs; + + @lombok.NonNull + @lombok.Singular(value = "meta") + Map meta; +} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 50341b48..d287ba7a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -21,7 +21,8 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.file.SdmxFile; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; @@ -72,7 +73,7 @@ protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key k result = copyOfKeys(super.loadData(entry, flowRef, key, true)); cache.put(loadDataKey, result); } - return Series.asCursor(result, key); + return SdmxRepository.asCursor(result, key); } return super.loadData(entry, flowRef, key, serieskeysonly); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 22b22b73..5faa8956 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -16,15 +16,21 @@ */ package be.nbb.sdmx.facade.repo; +import be.nbb.sdmx.facade.Obs; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; import java.io.IOException; +import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -75,7 +81,7 @@ public DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query Objects.requireNonNull(query); List col = data.get(flowRef); if (col != null) { - return Series.asCursor(col, query.getKey()); + return asCursor(col, query.getKey()); } throw new IOException("Data not found"); } @@ -104,10 +110,31 @@ public Builder data(@Nonnull DataflowRef flowRef, @Nonnull List list) { @Nonnull public Builder copyOf(@Nonnull DataflowRef flowRef, @Nonnull DataCursor cursor) throws IOException { - return data(flowRef, Series.copyOf(cursor)); + return data(flowRef, SdmxRepository.copyOf(cursor)); } } + @Nonnull + public static List copyOf(@Nonnull DataCursor cursor) throws IOException { + if (!cursor.nextSeries()) { + return Collections.emptyList(); + } + if (cursor instanceof SeriesCursor) { + return ((SeriesCursor) cursor).getRemainingItems(); + } + Series.Builder b = Series.builder(); + List result = new ArrayList<>(); + do { + result.add(getSeries(b, cursor)); + } while (cursor.nextSeries()); + return result; + } + + @Nonnull + public static DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { + return new SeriesCursor(list, ref); + } + // private static final class RepoConnection implements SdmxConnection { @@ -159,7 +186,7 @@ public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOExcepti Objects.requireNonNull(query); List col = data.get(flowRef); if (col != null) { - return Series.asCursor(col, query.getKey()); + return SdmxRepository.asCursor(col, query.getKey()); } throw new IOException("Data not found"); } @@ -184,5 +211,121 @@ private void checkState() throws IOException { private static Map toMap(List list, Function toKey) { return list.stream().collect(Collectors.toMap(toKey, Function.identity())); } - // + + private static Series getSeries(Series.Builder series, DataCursor cursor) throws IOException { + series.key(cursor.getSeriesKey()) + .frequency(cursor.getSeriesFrequency()) + .clearMeta() + .clearObs() + .meta(cursor.getSeriesAttributes()); + while (cursor.nextObs()) { + series.obs(Obs.of(cursor.getObsPeriod(), cursor.getObsValue())); + } + return series.build(); + } + + private static final class SeriesCursor implements DataCursor { + + private final List col; + private final Key ref; + private int i; + private int j; + private boolean closed; + private boolean hasSeries; + private boolean hasObs; + + SeriesCursor(List col, Key ref) { + this.col = col; + this.ref = ref; + this.i = -1; + this.j = -1; + this.closed = false; + this.hasSeries = false; + this.hasObs = false; + } + + @Override + public boolean nextSeries() throws IOException { + checkState(); + do { + i++; + j = -1; + } while (i < col.size() && !ref.contains(col.get(i).getKey())); + return hasSeries = (i < col.size()); + } + + @Override + public boolean nextObs() throws IOException { + checkSeriesState(); + j++; + return hasObs = (j < col.get(i).getObs().size()); + } + + @Override + public Key getSeriesKey() throws IOException { + checkSeriesState(); + return col.get(i).getKey(); + } + + @Override + public Frequency getSeriesFrequency() throws IOException { + checkSeriesState(); + return col.get(i).getFrequency(); + } + + @Override + public String getSeriesAttribute(String key) throws IOException { + checkSeriesState(); + Objects.requireNonNull(key); + return col.get(i).getMeta().get(key); + } + + @Override + public Map getSeriesAttributes() throws IOException { + checkSeriesState(); + return col.get(i).getMeta(); + } + + @Override + public LocalDateTime getObsPeriod() throws IOException { + checkObsState(); + return col.get(i).getObs().get(j).getPeriod(); + } + + @Override + public Double getObsValue() throws IOException { + checkObsState(); + return col.get(i).getObs().get(j).getValue(); + } + + @Override + public void close() throws IOException { + closed = true; + } + + List getRemainingItems() { + return col.subList(i, col.size()); + } + + private void checkState() throws IOException { + if (closed) { + throw new IOException("Cursor closed"); + } + } + + private void checkSeriesState() throws IOException, IllegalStateException { + checkState(); + if (!hasSeries) { + throw new IllegalStateException(); + } + } + + private void checkObsState() throws IOException, IllegalStateException { + checkSeriesState(); + if (!hasObs) { + throw new IllegalStateException(); + } + } + } + //} } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java deleted file mode 100644 index d510d34a..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/Series.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.repo; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.Key; -import java.io.IOException; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -@lombok.Value -@lombok.Builder(builderClassName = "Builder") -public class Series { - - @lombok.NonNull - Key key; - - @lombok.NonNull - Frequency frequency; - - @lombok.NonNull - @lombok.Singular(value = "obs") - List obs; - - @lombok.NonNull - @lombok.Singular(value = "meta") - Map meta; - - @Nonnull - public static List copyOf(@Nonnull DataCursor cursor) throws IOException { - if (!cursor.nextSeries()) { - return Collections.emptyList(); - } - - if (cursor instanceof SeriesCursor) { - return ((SeriesCursor) cursor).getRemainingItems(); - } - - Series.Builder b = builder(); - List result = new ArrayList<>(); - do { - result.add(getSeries(b, cursor)); - } while (cursor.nextSeries()); - return result; - } - - @Nonnull - public static DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { - return new SeriesCursor(list, ref); - } - - // - private static Series getSeries(Series.Builder series, DataCursor cursor) throws IOException { - series.key(cursor.getSeriesKey()) - .frequency(cursor.getSeriesFrequency()) - .clearMeta() - .clearObs() - .meta(cursor.getSeriesAttributes()); - while (cursor.nextObs()) { - series.obs(Obs.of(cursor.getObsPeriod(), cursor.getObsValue())); - } - return series.build(); - } - - private static final class SeriesCursor implements DataCursor { - - private final List col; - private final Key ref; - private int i; - private int j; - private boolean closed; - private boolean hasSeries; - private boolean hasObs; - - SeriesCursor(List col, Key ref) { - this.col = col; - this.ref = ref; - this.i = -1; - this.j = -1; - this.closed = false; - this.hasSeries = false; - this.hasObs = false; - } - - @Override - public boolean nextSeries() throws IOException { - checkState(); - do { - i++; - j = -1; - } while (i < col.size() && !ref.contains(col.get(i).getKey())); - return hasSeries = (i < col.size()); - } - - @Override - public boolean nextObs() throws IOException { - checkSeriesState(); - j++; - return hasObs = (j < col.get(i).getObs().size()); - } - - @Override - public Key getSeriesKey() throws IOException { - checkSeriesState(); - return col.get(i).getKey(); - } - - @Override - public Frequency getSeriesFrequency() throws IOException { - checkSeriesState(); - return col.get(i).getFrequency(); - } - - @Override - public String getSeriesAttribute(String key) throws IOException { - checkSeriesState(); - Objects.requireNonNull(key); - return col.get(i).getMeta().get(key); - } - - @Override - public Map getSeriesAttributes() throws IOException { - checkSeriesState(); - return col.get(i).getMeta(); - } - - @Override - public LocalDateTime getObsPeriod() throws IOException { - checkObsState(); - return col.get(i).getObs().get(j).getPeriod(); - } - - @Override - public Double getObsValue() throws IOException { - checkObsState(); - return col.get(i).getObs().get(j).getValue(); - } - - @Override - public void close() throws IOException { - closed = true; - } - - List getRemainingItems() { - return col.subList(i, col.size()); - } - - private void checkState() throws IOException { - if (closed) { - throw new IOException("Cursor closed"); - } - } - - private void checkSeriesState() throws IOException, IllegalStateException { - checkState(); - if (!hasSeries) { - throw new IllegalStateException(); - } - } - - private void checkObsState() throws IOException, IllegalStateException { - checkSeriesState(); - if (!hasObs) { - throw new IllegalStateException(); - } - } - } - // -} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index 1390949f..f37b4287 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -16,13 +16,24 @@ */ package be.nbb.sdmx.facade.repo; +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; import be.nbb.sdmx.facade.tck.DataCursorAssert; +import be.nbb.sdmx.facade.util.NoOpCursor; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import java.io.IOException; import java.time.LocalDateTime; import java.util.Collections; +import java.util.List; +import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; @@ -39,7 +50,7 @@ public void testCompliance() { @Test public void testDataCursorCompliance() { - DataCursorAssert.assertCompliance(() -> Series.asCursor(Collections.singletonList(series), Key.ALL)); + DataCursorAssert.assertCompliance(() -> SdmxRepository.asCursor(Collections.singletonList(series), Key.ALL)); } @Test @@ -47,6 +58,21 @@ public void testBuilder() { assertThat(SdmxRepository.builder().name("test").data(xyz, series).build().isSeriesKeysOnlySupported()).isTrue(); } + @Test + public void testCopyOf() throws IOException, XMLStreamException { + try (DataCursor c = NoOpCursor.noOp()) { + assertThat(SdmxRepository.copyOf(c)).isEmpty(); + } + + try (DataCursor c = SdmxRepository.asCursor(Collections.singletonList(series), Key.ALL)) { + assertThat(SdmxRepository.copyOf(c)).hasSize(1).element(0).isSameAs(series); + } + + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); + try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + assertThat(SdmxRepository.copyOf(o)).hasSize(120); + } + } private final DataflowRef xyz = DataflowRef.parse("XYZ"); private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); private final SdmxRepository repo = SdmxRepository.builder().name("test").data(xyz, series).build(); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java deleted file mode 100644 index 8eba9f90..00000000 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SeriesTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.repo; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.sdmx.facade.util.NoOpCursor; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import java.io.IOException; -import java.time.LocalDateTime; -import java.util.Collections; -import java.util.List; -import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.assertThat; -import org.junit.Test; - -/** - * - * @author Philippe Charles - */ -public class SeriesTest { - - @Test - public void testCopyOf() throws IOException, XMLStreamException { - try (DataCursor c = NoOpCursor.noOp()) { - assertThat(Series.copyOf(c)).isEmpty(); - } - - Series s1 = Series.builder().key(Key.of("hello")).frequency(Frequency.WEEKLY).obs(Obs.of(LocalDateTime.now(), 3.14)).build(); - try (DataCursor c = Series.asCursor(Collections.singletonList(s1), Key.ALL)) { - assertThat(Series.copyOf(c)).hasSize(1).element(0).isSameAs(s1); - } - - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); - try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { - assertThat(Series.copyOf(o)).hasSize(120); - } - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 7e1a2c6e..fa6f03ed 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -19,8 +19,9 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxMediaType; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; @@ -74,7 +75,7 @@ private InseeClient(URI endpoint, LanguagePriorityList langs) { @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return Series.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SdmxRepository.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -92,7 +93,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { - return Series.copyOf(cursor); + return SdmxRepository.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 100bf048..183ccdd6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -19,9 +19,10 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SdmxMediaType; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; @@ -187,7 +188,7 @@ private Sdmx21Client(URI endpoint, Sdmx21Config config, LanguagePriorityList lan @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return Series.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SdmxRepository.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -205,7 +206,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(new XMLEventStreamReader(r))) { - return Series.copyOf(cursor); + return SdmxRepository.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index 16cb851b..a57b361a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -22,7 +22,7 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.ObsParser; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index 6144fabe..63754d22 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -24,7 +24,7 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; import java.io.InputStreamReader; @@ -91,7 +91,7 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData20(dsd).get(f, xml.openReader())) { - return Series.copyOf(c); + return SdmxRepository.copyOf(c); } } @@ -111,7 +111,7 @@ private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorit List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData21(dsd).get(f, xml.openReader())) { - return Series.copyOf(c); + return SdmxRepository.copyOf(c); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index e23c703d..6f3b3550 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -20,8 +20,9 @@ import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import be.nbb.sdmx.facade.repo.Obs; -import be.nbb.sdmx.facade.repo.Series; +import be.nbb.sdmx.facade.Obs; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.ObsParser; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; @@ -52,7 +53,7 @@ public static void beforeClass() throws IOException { @Test public void test() throws IOException { try (PortableTimeSeriesCursor cursor = new PortableTimeSeriesCursor(DATA, ObsParser.standard())) { - assertThat(Series.copyOf(cursor)) + assertThat(SdmxRepository.copyOf(cursor)) .hasSize(120) .allMatch(o -> o.getFrequency().equals(Frequency.ANNUAL)) .element(0) From 92d79e7f408376e4174a8f5423668ffa16035e7e Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 26 Oct 2017 15:05:34 +0200 Subject: [PATCH 19/68] Added Stream API. --- .../java/internal/sdmx/SdmxQueryUtil.java | 2 +- .../demetra/dotstat/DotStatAccessorTest.java | 9 +- .../be/nbb/sdmx/facade/SdmxConnection.java | 6 +- .../file/CachedFileSdmxConnection.java | 4 +- .../internal/file/FileSdmxConnection.java | 10 +- .../internal/file/FileSdmxConnectionTest.java | 2 +- .../nbb/sdmx/facade/tck/ConnectionAssert.java | 19 +- .../nbb/sdmx/facade/repo/SdmxRepository.java | 232 ++++-------------- .../main/java/be/nbb/sdmx/facade/util/IO.java | 110 +++++++++ .../nbb/sdmx/facade/util/SeriesSupport.java | 192 +++++++++++++++ .../sdmx/facade/repo/SdmxRepositoryTest.java | 27 +- .../sdmx/facade/util/SeriesSupportTest.java | 85 +++++++ .../internal/connectors/CachedResource.java | 11 +- .../connectors/ConnectorsConnection.java | 10 +- .../connectors/drivers/InseeDriver.java | 6 +- .../connectors/drivers/Sdmx21Driver.java | 6 +- .../internal/connectors/FacadeResource.java | 5 +- .../PortableTimeSeriesCursorTest.java | 36 ++- 18 files changed, 511 insertions(+), 261 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index b1a598b5..5d063eb8 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -88,7 +88,7 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re private final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { - return new SdmxDataAdapter(key, conn.getData(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); + return new SdmxDataAdapter(key, conn.getDataCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); } private TsCursor computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index 3fc6771b..07db0743 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -17,7 +17,6 @@ package be.nbb.demetra.dotstat; import static be.nbb.demetra.dotstat.DotStatAccessor.getKey; -import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.DataflowRef; @@ -25,6 +24,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.Series; import internal.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import com.google.common.base.Joiner; @@ -100,10 +100,9 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { - try (DataCursor o = supplier.getConnection("NBB").getData(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true))) { - o.nextSeries(); - assertThat(o.getSeriesKey()).isEqualTo(Key.parse("LOCSTL04.AUS.M")); - } + assertThat(supplier.getConnection("NBB") + .getDataStream(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true)) + .map(Series::getKey)).contains(Key.parse("LOCSTL04.AUS.M")); } @Test diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java index ef4e858f..0260e4ba 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Set; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; @@ -39,7 +40,10 @@ public interface SdmxConnection extends Closeable { DataStructure getDataStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + DataCursor getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + + @Nonnull + Stream getDataStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index d287ba7a..011363f2 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -21,8 +21,8 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.file.SdmxFile; -import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; @@ -73,7 +73,7 @@ protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key k result = copyOfKeys(super.loadData(entry, flowRef, key, true)); cache.put(loadDataKey, result); } - return SdmxRepository.asCursor(result, key); + return SeriesSupport.asCursor(result, key); } return super.loadData(entry, flowRef, key, serieskeysonly); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 2d4644d0..55eed361 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -26,7 +26,9 @@ import be.nbb.sdmx.facade.DataQueryDetail; import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.io.IOException; @@ -34,6 +36,7 @@ import java.util.Collections; import java.util.Objects; import java.util.Set; +import java.util.stream.Stream; import javax.xml.stream.XMLInputFactory; /** @@ -83,7 +86,7 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { + final public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); Objects.requireNonNull(flowRef); Objects.requireNonNull(query); @@ -91,6 +94,11 @@ final public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOE return loadData(decode(), flowRef, query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); } + @Override + public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getDataCursor(flowRef, query)); + } + @Override final public boolean isSeriesKeysOnlySupported() { return true; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 385ed19e..ab0c14ae 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -55,7 +55,7 @@ public void testCompactData21() throws IOException { Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getData(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getDataCursor(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index a9d20ea4..d24f71fc 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -52,7 +52,8 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getData(ref, DataQuery.of(Key.ALL, false)), "getData(DataFlowRef, DataQuery)"); + assertState(s, supplier, o -> o.getDataCursor(ref, DataQuery.of(Key.ALL, false)), "getDataCursor(DataFlowRef, DataQuery)"); + assertState(s, supplier, o -> o.getDataStream(ref, DataQuery.of(Key.ALL, false)), "getDataStream(DataFlowRef, DataQuery)"); assertState(s, supplier, o -> o.getDataStructure(ref), "getDataStructure(DataFlowRef)"); assertState(s, supplier, o -> o.getDataflow(ref), "getDataflow(DataFlowRef)"); assertState(s, supplier, SdmxConnection::getDataflows, "getDataflows()"); @@ -60,12 +61,20 @@ public static void assertCompliance(SoftAssertions s, Callable s @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getData(null, DataQuery.of(Key.ALL, false))) - .as("Expecting 'getData(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getDataCursor(null, DataQuery.of(Key.ALL, false))) + .as("Expecting 'getDataCursor(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getData(ref, null)) - .as("Expecting 'getData(DataFlowRef, DataQuery)' to raise NPE when called with null query") + s.assertThatThrownBy(() -> conn.getDataCursor(ref, null)) + .as("Expecting 'getDataCursor(DataFlowRef, DataQuery)' to raise NPE when called with null query") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataStream(null, DataQuery.of(Key.ALL, false))) + .as("Expecting 'getDataStream(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataStream(ref, null)) + .as("Expecting 'getDataStream(DataFlowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getDataStructure(null)) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 5faa8956..a9742489 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -16,30 +16,24 @@ */ package be.nbb.sdmx.facade.repo; -import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; -import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * @@ -58,7 +52,7 @@ public class SdmxRepository { @lombok.NonNull @lombok.Singular - List dataflows; + Set dataflows; @lombok.NonNull Map> data; @@ -68,22 +62,23 @@ public class SdmxRepository { @Nonnull public SdmxConnection asConnection() { - return new RepoConnection( - toMap(dataStructures, DataStructure::getRef), - toMap(dataflows, Dataflow::getFlowRef), - data, - seriesKeysOnlySupported); + return new RepoConnection(this); } - @Nonnull - public DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException { + @Nullable + public DataCursor getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { Objects.requireNonNull(flowRef); Objects.requireNonNull(query); - List col = data.get(flowRef); - if (col != null) { - return asCursor(col, query.getKey()); - } - throw new IOException("Data not found"); + List result = data.get(flowRef); + return result != null ? SeriesSupport.asCursor(result, query.getKey()) : null; + } + + @Nullable + public Stream getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { + Objects.requireNonNull(flowRef); + Objects.requireNonNull(query); + List result = data.get(flowRef); + return result != null ? result.stream().filter(o -> query.getKey().contains(o.getKey())) : null; } public static final class Builder { @@ -110,90 +105,71 @@ public Builder data(@Nonnull DataflowRef flowRef, @Nonnull List list) { @Nonnull public Builder copyOf(@Nonnull DataflowRef flowRef, @Nonnull DataCursor cursor) throws IOException { - return data(flowRef, SdmxRepository.copyOf(cursor)); - } - } - - @Nonnull - public static List copyOf(@Nonnull DataCursor cursor) throws IOException { - if (!cursor.nextSeries()) { - return Collections.emptyList(); + return data(flowRef, SeriesSupport.copyOf(cursor)); } - if (cursor instanceof SeriesCursor) { - return ((SeriesCursor) cursor).getRemainingItems(); - } - Series.Builder b = Series.builder(); - List result = new ArrayList<>(); - do { - result.add(getSeries(b, cursor)); - } while (cursor.nextSeries()); - return result; - } - - @Nonnull - public static DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { - return new SeriesCursor(list, ref); } // private static final class RepoConnection implements SdmxConnection { - private final Map dataStructures; - private final Map dataflows; - private final Map> data; - private final boolean seriesKeysOnlySupported; + private final SdmxRepository repo; private boolean closed; - private RepoConnection(Map dataStructures, Map dataflows, Map> data, boolean seriesKeysOnlySupported) { - this.dataStructures = dataStructures; - this.dataflows = dataflows; - this.data = data; - this.seriesKeysOnlySupported = seriesKeysOnlySupported; + private RepoConnection(SdmxRepository repo) { + this.repo = repo; this.closed = false; } @Override public Set getDataflows() throws IOException { checkState(); - return new HashSet<>(dataflows.values()); + return repo.getDataflows(); } @Override public Dataflow getDataflow(DataflowRef flowRef) throws IOException { checkState(); Objects.requireNonNull(flowRef); - Dataflow result = dataflows.get(flowRef); - if (result != null) { - return result; - } - throw new IOException("Dataflow not found"); + return repo.getDataflows() + .stream() + .filter(o -> flowRef.equals(o.getFlowRef())) + .findFirst() + .orElseThrow(() -> new IOException("Dataflow not found")); } @Override public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { + Dataflow flow = getDataflow(flowRef); + return repo.getDataStructures() + .stream() + .filter(o -> flow.getDataStructureRef().equals(o.getRef())) + .findFirst() + .orElseThrow(() -> new IOException("DataStructure not found")); + } + + @Override + public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - DataStructure result = dataStructures.get(getDataflow(flowRef).getDataStructureRef()); + DataCursor result = repo.getCursor(flowRef, query); if (result != null) { return result; } - throw new IOException("DataStructure not found"); + throw new IOException("Data not found"); } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { + public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - Objects.requireNonNull(flowRef); - Objects.requireNonNull(query); - List col = data.get(flowRef); - if (col != null) { - return SdmxRepository.asCursor(col, query.getKey()); + Stream result = repo.getStream(flowRef, query); + if (result != null) { + return result; } throw new IOException("Data not found"); } @Override public boolean isSeriesKeysOnlySupported() { - return seriesKeysOnlySupported; + return repo.isSeriesKeysOnlySupported(); } @Override @@ -207,125 +183,5 @@ private void checkState() throws IOException { } } } - - private static Map toMap(List list, Function toKey) { - return list.stream().collect(Collectors.toMap(toKey, Function.identity())); - } - - private static Series getSeries(Series.Builder series, DataCursor cursor) throws IOException { - series.key(cursor.getSeriesKey()) - .frequency(cursor.getSeriesFrequency()) - .clearMeta() - .clearObs() - .meta(cursor.getSeriesAttributes()); - while (cursor.nextObs()) { - series.obs(Obs.of(cursor.getObsPeriod(), cursor.getObsValue())); - } - return series.build(); - } - - private static final class SeriesCursor implements DataCursor { - - private final List col; - private final Key ref; - private int i; - private int j; - private boolean closed; - private boolean hasSeries; - private boolean hasObs; - - SeriesCursor(List col, Key ref) { - this.col = col; - this.ref = ref; - this.i = -1; - this.j = -1; - this.closed = false; - this.hasSeries = false; - this.hasObs = false; - } - - @Override - public boolean nextSeries() throws IOException { - checkState(); - do { - i++; - j = -1; - } while (i < col.size() && !ref.contains(col.get(i).getKey())); - return hasSeries = (i < col.size()); - } - - @Override - public boolean nextObs() throws IOException { - checkSeriesState(); - j++; - return hasObs = (j < col.get(i).getObs().size()); - } - - @Override - public Key getSeriesKey() throws IOException { - checkSeriesState(); - return col.get(i).getKey(); - } - - @Override - public Frequency getSeriesFrequency() throws IOException { - checkSeriesState(); - return col.get(i).getFrequency(); - } - - @Override - public String getSeriesAttribute(String key) throws IOException { - checkSeriesState(); - Objects.requireNonNull(key); - return col.get(i).getMeta().get(key); - } - - @Override - public Map getSeriesAttributes() throws IOException { - checkSeriesState(); - return col.get(i).getMeta(); - } - - @Override - public LocalDateTime getObsPeriod() throws IOException { - checkObsState(); - return col.get(i).getObs().get(j).getPeriod(); - } - - @Override - public Double getObsValue() throws IOException { - checkObsState(); - return col.get(i).getObs().get(j).getValue(); - } - - @Override - public void close() throws IOException { - closed = true; - } - - List getRemainingItems() { - return col.subList(i, col.size()); - } - - private void checkState() throws IOException { - if (closed) { - throw new IOException("Cursor closed"); - } - } - - private void checkSeriesState() throws IOException, IllegalStateException { - checkState(); - if (!hasSeries) { - throw new IllegalStateException(); - } - } - - private void checkObsState() throws IOException, IllegalStateException { - checkSeriesState(); - if (!hasObs) { - throw new IllegalStateException(); - } - } - } //} } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java new file mode 100644 index 00000000..9aa1aaa5 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java @@ -0,0 +1,110 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class IO { + + @FunctionalInterface + public interface Supplier { + + T getWithIO() throws IOException; + } + + @FunctionalInterface + public interface Function { + + R applyWithIO(T t) throws IOException; + } + + @Nonnull + public Stream stream(IO.Supplier supplier, IO.Function> stream) throws IOException { + T resource = supplier.getWithIO(); + try { + return stream.applyWithIO(resource).onClose(asUncheckedRunnable(resource)); + } catch (Error | RuntimeException e) { + try { + resource.close(); + } catch (IOException ex) { + try { + e.addSuppressed(ex); + } catch (Throwable ignore) { + } + } + throw e; + } + } + + @Nonnull + public Stream streamNonnull(@Nonnull IO.Supplier nextSupplier) { + Iterator iter = new Iterator() { + T nextElement = null; + + @Override + public boolean hasNext() { + if (nextElement != null) { + return true; + } else { + try { + nextElement = nextSupplier.getWithIO(); + return (nextElement != null); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Override + public T next() { + if (nextElement != null || hasNext()) { + T line = nextElement; + nextElement = null; + return line; + } else { + throw new NoSuchElementException(); + } + } + }; + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iter, Spliterator.ORDERED | Spliterator.NONNULL), false); + } + + private static Runnable asUncheckedRunnable(Closeable c) { + return () -> { + try { + c.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java new file mode 100644 index 00000000..4ba7b32b --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -0,0 +1,192 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.Obs; +import be.nbb.sdmx.facade.Series; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class SeriesSupport { + + @Nonnull + public List copyOf(@Nonnull DataCursor cursor) throws IOException { + if (!cursor.nextSeries()) { + return Collections.emptyList(); + } + + if (cursor instanceof SeriesCursor) { + return ((SeriesCursor) cursor).getRemainingItems(); + } + + Series.Builder b = Series.builder(); + List result = new ArrayList<>(); + do { + result.add(getSeries(b, cursor)); + } while (cursor.nextSeries()); + return result; + } + + @Nonnull + public DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { + Objects.requireNonNull(list); + Objects.requireNonNull(ref); + return new SeriesCursor(list, ref); + } + + @Nonnull + public Stream asStream(@Nonnull IO.Supplier supplier) throws IOException { + return IO.stream(supplier, SeriesSupport::getDataStream); + } + + @SuppressWarnings("null") + private Stream getDataStream(DataCursor cursor) { + Series.Builder builder = Series.builder(); + return IO.streamNonnull(() -> cursor.nextSeries() ? getSeries(builder, cursor) : null); + } + + private Series getSeries(Series.Builder builder, DataCursor cursor) throws IOException { + builder.key(cursor.getSeriesKey()) + .frequency(cursor.getSeriesFrequency()) + .clearMeta() + .clearObs() + .meta(cursor.getSeriesAttributes()); + while (cursor.nextObs()) { + builder.obs(Obs.of(cursor.getObsPeriod(), cursor.getObsValue())); + } + return builder.build(); + } + + private static final class SeriesCursor implements DataCursor { + + private final List col; + private final Key ref; + private int i; + private int j; + private boolean closed; + private boolean hasSeries; + private boolean hasObs; + + SeriesCursor(List col, Key ref) { + this.col = col; + this.ref = ref; + this.i = -1; + this.j = -1; + this.closed = false; + this.hasSeries = false; + this.hasObs = false; + } + + @Override + public boolean nextSeries() throws IOException { + checkState(); + do { + i++; + j = -1; + } while (i < col.size() && !ref.contains(col.get(i).getKey())); + return hasSeries = (i < col.size()); + } + + @Override + public boolean nextObs() throws IOException { + checkSeriesState(); + j++; + return hasObs = (j < col.get(i).getObs().size()); + } + + @Override + public Key getSeriesKey() throws IOException { + checkSeriesState(); + return col.get(i).getKey(); + } + + @Override + public Frequency getSeriesFrequency() throws IOException { + checkSeriesState(); + return col.get(i).getFrequency(); + } + + @Override + public String getSeriesAttribute(String key) throws IOException { + checkSeriesState(); + Objects.requireNonNull(key); + return col.get(i).getMeta().get(key); + } + + @Override + public Map getSeriesAttributes() throws IOException { + checkSeriesState(); + return col.get(i).getMeta(); + } + + @Override + public LocalDateTime getObsPeriod() throws IOException { + checkObsState(); + return col.get(i).getObs().get(j).getPeriod(); + } + + @Override + public Double getObsValue() throws IOException { + checkObsState(); + return col.get(i).getObs().get(j).getValue(); + } + + @Override + public void close() throws IOException { + closed = true; + } + + List getRemainingItems() { + return col.subList(i, col.size()); + } + + private void checkState() throws IOException { + if (closed) { + throw new IOException("Cursor closed"); + } + } + + private void checkSeriesState() throws IOException, IllegalStateException { + checkState(); + if (!hasSeries) { + throw new IllegalStateException(); + } + } + + private void checkObsState() throws IOException, IllegalStateException { + checkSeriesState(); + if (!hasObs) { + throw new IllegalStateException(); + } + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index f37b4287..fc65ed40 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -16,24 +16,16 @@ */ package be.nbb.sdmx.facade.repo; -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import be.nbb.sdmx.facade.util.NoOpCursor; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import java.io.IOException; +import be.nbb.sdmx.facade.util.SeriesSupport; import java.time.LocalDateTime; import java.util.Collections; -import java.util.List; -import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; @@ -50,7 +42,7 @@ public void testCompliance() { @Test public void testDataCursorCompliance() { - DataCursorAssert.assertCompliance(() -> SdmxRepository.asCursor(Collections.singletonList(series), Key.ALL)); + DataCursorAssert.assertCompliance(() -> SeriesSupport.asCursor(Collections.singletonList(series), Key.ALL)); } @Test @@ -58,21 +50,6 @@ public void testBuilder() { assertThat(SdmxRepository.builder().name("test").data(xyz, series).build().isSeriesKeysOnlySupported()).isTrue(); } - @Test - public void testCopyOf() throws IOException, XMLStreamException { - try (DataCursor c = NoOpCursor.noOp()) { - assertThat(SdmxRepository.copyOf(c)).isEmpty(); - } - - try (DataCursor c = SdmxRepository.asCursor(Collections.singletonList(series), Key.ALL)) { - assertThat(SdmxRepository.copyOf(c)).hasSize(1).element(0).isSameAs(series); - } - - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); - try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { - assertThat(SdmxRepository.copyOf(o)).hasSize(120); - } - } private final DataflowRef xyz = DataflowRef.parse("XYZ"); private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); private final SdmxRepository repo = SdmxRepository.builder().name("test").data(xyz, series).build(); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java new file mode 100644 index 00000000..304d0091 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -0,0 +1,85 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.Obs; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.tck.DataCursorAssert; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.List; +import javax.xml.stream.XMLStreamException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SeriesSupportTest { + + @Test + @SuppressWarnings("null") + public void testAsCursor() { + assertThatThrownBy(() -> SeriesSupport.asCursor(null, Key.ALL)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SeriesSupport.asCursor(Collections.emptyList(), null)).isInstanceOf(NullPointerException.class); + + DataCursorAssert.assertCompliance(() -> SeriesSupport.asCursor(Collections.emptyList(), Key.ALL)); + } + + @Test + @SuppressWarnings("null") + public void testAsStream() { + assertThatThrownBy(() -> SeriesSupport.asStream(null)).isInstanceOf(NullPointerException.class); + + assertThatThrownBy(() -> { + DataCursor cursor = NoOpCursor.noOp(); + SeriesSupport.asStream(() -> cursor).count(); + cursor.getSeriesKey(); + }).isInstanceOf(IllegalStateException.class); + } + + @Test + @SuppressWarnings("null") + public void testCopyOf() throws IOException, XMLStreamException { + assertThatThrownBy(() -> SeriesSupport.copyOf(null)).isInstanceOf(NullPointerException.class); + + try (DataCursor c = NoOpCursor.noOp()) { + assertThat(SeriesSupport.copyOf(c)).isEmpty(); + } + + try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(series), Key.ALL)) { + assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); + } + + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); + try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + assertThat(SeriesSupport.copyOf(o)).hasSize(120); + } + } + + private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java index 6d8807e9..abb25fc4 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java @@ -79,9 +79,14 @@ public DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOExcepti @Override public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return serieskeysonly - ? loadKeysOnlyWithCache(flowRef, key).getData(flowRef, DataQuery.of(key, true)) - : super.loadData(flowRef, key, serieskeysonly); + if (!serieskeysonly) { + return super.loadData(flowRef, key, serieskeysonly); + } + DataCursor result = loadKeysOnlyWithCache(flowRef, key).getCursor(flowRef, DataQuery.of(key, true)); + if (result != null) { + return result; + } + throw new IOException("Data not found"); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java index 86eebf06..3f0fb777 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java @@ -24,10 +24,13 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; /** @@ -82,7 +85,7 @@ public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOException { + public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); if (serieskeysonly && !isSeriesKeysOnlySupported()) { @@ -91,6 +94,11 @@ public DataCursor getData(DataflowRef flowRef, DataQuery query) throws IOExcepti return resource.loadData(flowRef, query.getKey(), serieskeysonly); } + @Override + public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getDataCursor(flowRef, query)); + } + @Override public boolean isSeriesKeysOnlySupported() { return resource.isSeriesKeysOnlySupported(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index fa6f03ed..71e20d68 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -19,11 +19,11 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxMediaType; +import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; @@ -75,7 +75,7 @@ private InseeClient(URI endpoint, LanguagePriorityList langs) { @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return SdmxRepository.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SeriesSupport.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -93,7 +93,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { - return SdmxRepository.copyOf(cursor); + return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 183ccdd6..b24add24 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -19,11 +19,11 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SdmxMediaType; +import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; @@ -188,7 +188,7 @@ private Sdmx21Client(URI endpoint, Sdmx21Config config, LanguagePriorityList lan @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return SdmxRepository.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SeriesSupport.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); } @Override @@ -206,7 +206,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(new XMLEventStreamReader(r))) { - return SdmxRepository.copyOf(cursor); + return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index 63754d22..3381cb1f 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -25,6 +25,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; import java.io.InputStreamReader; @@ -91,7 +92,7 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData20(dsd).get(f, xml.openReader())) { - return SdmxRepository.copyOf(c); + return SeriesSupport.copyOf(c); } } @@ -111,7 +112,7 @@ private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorit List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { try (DataCursor c = SdmxXmlStreams.genericData21(dsd).get(f, xml.openReader())) { - return SdmxRepository.copyOf(c); + return SeriesSupport.copyOf(c); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index 6f3b3550..faa9234d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -21,9 +21,8 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.Obs; -import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.util.SeriesSupport; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; @@ -33,7 +32,6 @@ import org.junit.BeforeClass; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThat; /** * @@ -52,23 +50,21 @@ public static void beforeClass() throws IOException { @Test public void test() throws IOException { - try (PortableTimeSeriesCursor cursor = new PortableTimeSeriesCursor(DATA, ObsParser.standard())) { - assertThat(SdmxRepository.copyOf(cursor)) - .hasSize(120) - .allMatch(o -> o.getFrequency().equals(Frequency.ANNUAL)) - .element(0) - .satisfies(o -> { - assertThat(o.getKey()).isEqualTo(Key.parse("A.DEU.1.0.319.0.UBLGE")); - assertThat(o.getMeta()) - .hasSize(3) - .containsEntry("EXT_UNIT", "Percentage of GDP at market prices (excessive deficit procedure)") - .isNotEmpty(); - assertThat(o.getObs()) - .hasSize(25) - .startsWith(Obs.of(LocalDate.of(1991, 1, 1).atStartOfDay(), -2.8574221)) - .endsWith(Obs.of(LocalDate.of(2015, 1, 1).atStartOfDay(), -0.1420473)); - }); - } + assertThat(SeriesSupport.asStream(() -> new PortableTimeSeriesCursor(DATA, ObsParser.standard()))) + .hasSize(120) + .allMatch(o -> o.getFrequency().equals(Frequency.ANNUAL)) + .element(0) + .satisfies(o -> { + assertThat(o.getKey()).isEqualTo(Key.parse("A.DEU.1.0.319.0.UBLGE")); + assertThat(o.getMeta()) + .hasSize(3) + .containsEntry("EXT_UNIT", "Percentage of GDP at market prices (excessive deficit procedure)") + .isNotEmpty(); + assertThat(o.getObs()) + .hasSize(25) + .startsWith(Obs.of(LocalDate.of(1991, 1, 1).atStartOfDay(), -2.8574221)) + .endsWith(Obs.of(LocalDate.of(2015, 1, 1).atStartOfDay(), -0.1420473)); + }); } @Test From 3ff5e30669219c35e8f97ea4e0c92a720771468d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 27 Oct 2017 10:14:37 +0200 Subject: [PATCH 20/68] Refactored api methods by removing useless prefix. --- .../nbb/demetra/dotstat/DotStatAccessor.java | 10 +++--- .../nbb/demetra/dotstat/DotStatProvider.java | 6 ++-- .../java/internal/sdmx/SdmxCubeAccessor.java | 16 ++++----- .../java/internal/sdmx/SdmxCubeItems.java | 2 +- .../java/internal/sdmx/SdmxQueryUtil.java | 8 ++--- .../demetra/dotstat/DotStatAccessorTest.java | 4 +-- .../demetra/sdmx/web/SdmxWebProviderTest.java | 6 ++-- .../internal/connectors/TestResource.java | 8 ++--- .../internal/sdmx/SdmxAutoCompletion.java | 10 +++--- .../be/nbb/sdmx/facade/DataStructureRef.java | 6 ++-- .../java/be/nbb/sdmx/facade/Dataflow.java | 7 ++-- .../java/be/nbb/sdmx/facade/DataflowRef.java | 8 ++--- .../java/be/nbb/sdmx/facade/ResourceRef.java | 2 +- .../be/nbb/sdmx/facade/SdmxConnection.java | 10 +++--- .../main/java/be/nbb/sdmx/facade/Series.java | 2 +- .../main/java/internal/util/ResourceRefs.java | 2 +- .../java/be/nbb/sdmx/facade/FlowRefTest.java | 9 ++--- .../file/CachedFileSdmxConnection.java | 2 +- .../internal/file/FileSdmxConnection.java | 14 ++++---- .../internal/file/FileSdmxConnectionTest.java | 6 ++-- .../nbb/sdmx/facade/tck/ConnectionAssert.java | 34 +++++++++---------- .../nbb/sdmx/facade/repo/SdmxRepository.java | 16 ++++----- .../nbb/sdmx/facade/util/SeriesSupport.java | 4 +-- .../sdmx/facade/repo/SdmxRepositoryTest.java | 2 +- .../sdmx/facade/util/SeriesSupportTest.java | 2 +- .../connectors/ConnectorsConnection.java | 18 +++++----- .../connectors/GenericSDMXClientResource.java | 4 +-- .../main/java/internal/connectors/Util.java | 18 +++++----- .../connectors/drivers/InseeDriver.java | 2 +- .../connectors/drivers/Sdmx21Driver.java | 2 +- .../connectors/ConnectorsResource.java | 18 +++++----- .../internal/connectors/FacadeResource.java | 4 +-- .../PortableTimeSeriesCursorTest.java | 2 +- 33 files changed, 134 insertions(+), 130 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java index 69253a1b..9ee1099f 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java @@ -100,7 +100,7 @@ public DbAccessor memoize() { } private static List getAllSeries(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key colKey = converter.convert(ref); try (TsCursor cursor = SdmxQueryUtil.getAllSeries(conn, flowRef, colKey, SdmxQueryUtil.NO_LABEL)) { @@ -113,7 +113,7 @@ private static List getAllSeries(SdmxConnection conn, DataflowRef flowR } private static List getAllSeriesWithData(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key colKey = converter.convert(ref); try (TsCursor cursor = SdmxQueryUtil.getAllSeriesWithData(conn, flowRef, colKey, SdmxQueryUtil.NO_LABEL)) { @@ -126,15 +126,15 @@ private static List getAllSeriesWithData(SdmxConnection conn, Dataflow } private static DbSeries getSeriesWithData(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key seriesKey = converter.convert(ref); return new DbSeries(ref, SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey)); } private static List getChildren(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); - int dimensionPosition = dimensionById(conn.getDataStructure(flowRef)).get(ref.getColumn(ref.getLevel())).getPosition(); + Converter converter = getConverter(conn.getStructure(flowRef), ref); + int dimensionPosition = dimensionById(conn.getStructure(flowRef)).get(ref.getColumn(ref.getLevel())).getPosition(); return SdmxQueryUtil.getChildren(conn, flowRef, converter.convert(ref), dimensionPosition); } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java index 397a8d13..c2587bf8 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java @@ -85,7 +85,7 @@ public String getDisplayName(DataSource dataSource) { DotStatBean bean = decodeBean(dataSource); if (!displayCodes) { try (SdmxConnection conn = connect(bean.getDbName())) { - return String.format("%s ~ %s", bean.getDbName(), conn.getDataflow(bean.getFlowRef()).getLabel()); + return String.format("%s ~ %s", bean.getDbName(), conn.getFlow(bean.getFlowRef()).getLabel()); } catch (IOException | RuntimeException ex) { } } @@ -96,7 +96,7 @@ public String getDisplayName(DataSource dataSource) { public String getDisplayName(DataSet dataSet) { DotStatBean bean = decodeBean(dataSet.getDataSource()); try (SdmxConnection conn = connect(bean.getDbName())) { - DataStructure dfs = conn.getDataStructure(bean.getFlowRef()); + DataStructure dfs = conn.getStructure(bean.getFlowRef()); Key.Builder b = Key.builder(dfs); for (Dimension o : dfs.getDimensions()) { String value = dataSet.get(o.getId()); @@ -117,7 +117,7 @@ public String getDisplayNodeName(DataSet dataSet) { if (!displayCodes) { DotStatBean bean = decodeBean(dataSet.getDataSource()); try (SdmxConnection conn = connect(bean.getDbName())) { - DataStructure dfs = conn.getDataStructure(bean.getFlowRef()); + DataStructure dfs = conn.getStructure(bean.getFlowRef()); for (Dimension o : dfs.getDimensions()) { if (o.getId().equals(nodeDim.getKey())) { return o.getCodes().get(nodeDim.getValue()); diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index 08081ad3..ec4c56d7 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -117,7 +117,7 @@ public IteratorWithIO getChildren(CubeId ref) throws IOException { @Override public String getDisplayName() throws IOException { try (SdmxConnection conn = supplier.getConnection(source, languages)) { - return String.format("%s ~ %s", source, conn.getDataflow(flowRef).getLabel()); + return String.format("%s ~ %s", source, conn.getFlow(flowRef).getLabel()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } @@ -129,7 +129,7 @@ public String getDisplayName(CubeId id) throws IOException { return "All"; } try (SdmxConnection conn = supplier.getConnection(source, languages)) { - Map dimensionById = dimensionById(conn.getDataStructure(flowRef)); + Map dimensionById = dimensionById(conn.getStructure(flowRef)); return getKey(dimensionById, id).toString(); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); @@ -142,7 +142,7 @@ public String getDisplayNodeName(CubeId id) throws IOException { return "All"; } try (SdmxConnection conn = supplier.getConnection(source, languages)) { - Map dimensionById = dimensionById(conn.getDataStructure(flowRef)); + Map dimensionById = dimensionById(conn.getStructure(flowRef)); return getDisplayNodeName(dimensionById, id); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); @@ -151,7 +151,7 @@ public String getDisplayNodeName(CubeId id) throws IOException { // private static TsCursor getAllSeriesCursor(SdmxConnection conn, DataflowRef flowRef, CubeId ref, String labelAttribute) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key colKey = converter.convert(ref); TsCursor cursor = SdmxQueryUtil.getAllSeries(conn, flowRef, colKey, labelAttribute); @@ -160,7 +160,7 @@ private static TsCursor getAllSeriesCursor(SdmxConnection conn, Dataflow } private static TsCursor getAllSeriesWithDataCursor(SdmxConnection conn, DataflowRef flowRef, CubeId ref, String labelAttribute) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key colKey = converter.convert(ref); TsCursor cursor = SdmxQueryUtil.getAllSeriesWithData(conn, flowRef, colKey, labelAttribute); @@ -169,7 +169,7 @@ private static TsCursor getAllSeriesWithDataCursor(SdmxConnection conn, } private static TsCursor getSeriesWithDataCursor(SdmxConnection conn, DataflowRef flowRef, CubeId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); + Converter converter = getConverter(conn.getStructure(flowRef), ref); Key seriesKey = converter.convert(ref); TsCursor cursor = TsCursor.singleton(seriesKey, SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey)); @@ -178,8 +178,8 @@ private static TsCursor getSeriesWithDataCursor(SdmxConnection conn, Dat } private static IteratorWithIO getChildren(SdmxConnection conn, DataflowRef flowRef, CubeId ref) throws IOException { - Converter converter = getConverter(conn.getDataStructure(flowRef), ref); - int dimensionPosition = dimensionById(conn.getDataStructure(flowRef)).get(ref.getDimensionId(ref.getLevel())).getPosition(); + Converter converter = getConverter(conn.getStructure(flowRef), ref); + int dimensionPosition = dimensionById(conn.getStructure(flowRef)).get(ref.getDimensionId(ref.getLevel())).getPosition(); List children = SdmxQueryUtil.getChildren(conn, flowRef, converter.convert(ref), dimensionPosition); return IteratorWithIO.from(children.iterator()).transform(ref::child); } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index c74cb5b2..3517e7d3 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -42,7 +42,7 @@ public class SdmxCubeItems { public static List getDefaultDimIds(SdmxConnectionSupplier supplier, LanguagePriorityList languages, String source, DataflowRef flow) throws IOException { try (SdmxConnection conn = supplier.getConnection(source, languages)) { - return conn.getDataStructure(flow).getDimensions().stream().map(Dimension::getId).collect(Collectors.toList()); + return conn.getStructure(flow).getDimensions().stream().map(Dimension::getId).collect(Collectors.toList()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index 5d063eb8..e5af72b0 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -81,23 +81,23 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re } } - return computeAllPossibleChildren(dimensionByIndex(conn.getDataStructure(flowRef)), dimensionPosition); + return computeAllPossibleChildren(dimensionByIndex(conn.getStructure(flowRef)), dimensionPosition); } // private final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { - return new SdmxDataAdapter(key, conn.getDataCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); + return new SdmxDataAdapter(key, conn.getCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); } private TsCursor computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { - List list = computeAllPossibleSeries(dimensionByIndex(conn.getDataStructure(flowRef)), key); + List list = computeAllPossibleSeries(dimensionByIndex(conn.getStructure(flowRef)), key); return TsCursor.from(list.iterator()); } private TsCursor computeKeysAndRequestData(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { - List list = computeAllPossibleSeries(dimensionByIndex(conn.getDataStructure(flowRef)), key); + List list = computeAllPossibleSeries(dimensionByIndex(conn.getStructure(flowRef)), key); Map dataByKey = dataByKey(conn, flowRef, key); return TsCursor.from(list.iterator(), o -> dataByKey.getOrDefault(o, MISSING_DATA)); } diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index 07db0743..248af315 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -80,7 +80,7 @@ private static DbSetId ecbRoot() { @Test public void testGetKey() throws Exception { - DataStructure dfs = supplier.getConnection("NBB").getDataStructure(DataflowRef.of("NBB", "TEST_DATASET", null)); + DataStructure dfs = supplier.getConnection("NBB").getStructure(DataflowRef.of("NBB", "TEST_DATASET", null)); Map dimById = DotStatAccessor.dimensionById(dfs); // default ordering of dimensions @@ -101,7 +101,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { assertThat(supplier.getConnection("NBB") - .getDataStream(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true)) + .getStream(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true)) .map(Series::getKey)).contains(Key.parse("LOCSTL04.AUS.M")); } diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java index 6427fad8..17ea4c4f 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java @@ -129,9 +129,9 @@ private static SdmxRepository getCustomRepo() { .name("world") .dataStructure(conjStruct) .dataflow(conj) - .data(conj.getFlowRef(), Series.builder().key(Key.of("BE", "IND")).frequency(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 1)).build()) - .data(conj.getFlowRef(), Series.builder().key(Key.of("BE", "XXX")).frequency(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 2)).build()) - .data(conj.getFlowRef(), Series.builder().key(Key.of("FR", "IND")).frequency(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 3)).build()) + .data(conj.getRef(), Series.builder().key(Key.of("BE", "IND")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 1)).build()) + .data(conj.getRef(), Series.builder().key(Key.of("BE", "XXX")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 2)).build()) + .data(conj.getRef(), Series.builder().key(Key.of("FR", "IND")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 3)).build()) .build(); } diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 043e27de..663bb39b 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -55,7 +55,7 @@ public static final SdmxRepository nbb() { result.dataflows(parse(SdmxSource.NBB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser()) .stream() .map(TestResource::toDataFlow) - .filter(o -> o.getFlowRef().equals(flowRef)) + .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dsd = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).get(SdmxSource.XIF, SdmxSource.NBB_DATA.openReader())) { @@ -79,8 +79,8 @@ public static final SdmxRepository ecb() { result.dataStructures(dataStructures.values()); DataflowRef flowRef = DataflowRef.of("ECB", "AME", "1.0"); result.dataflows(parse(SdmxSource.ECB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()).stream() - .map(Util::toDataflow) - .filter(o -> o.getFlowRef().equals(flowRef)) + .map(Util::toFlow) + .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dfs = dataStructures.get(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).get(SdmxSource.XIF, SdmxSource.ECB_DATA.openReader())) { @@ -97,7 +97,7 @@ public static final SdmxRepository ecb() { private static Map toDataStructures(List input) { return input.stream() - .map(Util::toDataStructure) + .map(Util::toStructure) .collect(Collectors.toMap(DataStructure::getRef, Function.identity())); } diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java index 954fe187..cd7ce915 100644 --- a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java @@ -67,13 +67,13 @@ public AutoCompletionSource onFlows(SdmxConnectionSupplier supplier, LanguagePri .builder(o -> loadFlows(supplier, languages, source)) .behavior(o -> canLoadFlows(source) ? ASYNC : NONE) .postProcessor(SdmxAutoCompletion::filterAndSortFlows) - .valueToString(o -> o.getFlowRef().toString()) + .valueToString(o -> o.getRef().toString()) .cache(cache, o -> getFlowCacheKey(source, languages), SYNC) .build(); } public ListCellRenderer getFlowsRenderer() { - return CustomListCellRenderer.of(Dataflow::getLabel, o -> o.getFlowRef().toString()); + return CustomListCellRenderer.of(Dataflow::getLabel, o -> o.getRef().toString()); } public AutoCompletionSource onDimensions(SdmxConnectionSupplier supplier, LanguagePriorityList languages, Supplier source, Supplier flow, ConcurrentMap cache) { @@ -117,7 +117,7 @@ private boolean canLoadFlows(Supplier source) { private List loadFlows(SdmxConnectionSupplier supplier, LanguagePriorityList languages, Supplier source) throws IOException { try (SdmxConnection c = supplier.getConnection(source.get(), languages)) { - return new ArrayList<>(c.getDataflows()); + return new ArrayList<>(c.getFlows()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } @@ -126,7 +126,7 @@ private List loadFlows(SdmxConnectionSupplier supplier, LanguagePriori private List filterAndSortFlows(List values, String term) { Predicate filter = ExtAutoCompletionSource.basicFilter(term); return values.stream() - .filter(o -> filter.test(o.getLabel()) || filter.test(o.getFlowRef().getId())) + .filter(o -> filter.test(o.getLabel()) || filter.test(o.getRef().getId())) .sorted(Comparator.comparing(Dataflow::getLabel)) .collect(Collectors.toList()); } @@ -141,7 +141,7 @@ private boolean canLoadDimensions(Supplier source, Supplier flow private List loadDimensions(SdmxConnectionSupplier supplier, LanguagePriorityList languages, Supplier source, Supplier flow) throws IOException { try (SdmxConnection c = supplier.getConnection(source.get(), languages)) { - return new ArrayList<>(c.getDataStructure(DataflowRef.parse(flow.get())).getDimensions()); + return new ArrayList<>(c.getStructure(DataflowRef.parse(flow.get())).getDimensions()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java index de713f7a..b7130ab5 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java @@ -31,7 +31,7 @@ public final class DataStructureRef implements ResourceRef { @lombok.NonNull - private String agencyId; + private String agency; @lombok.NonNull private String id; @@ -50,7 +50,7 @@ public static DataStructureRef parse(@Nonnull String input) throws IllegalArgume } @Nonnull - public static DataStructureRef of(@Nullable String agencyId, @Nonnull String flowId, @Nullable String version) throws IllegalArgumentException { - return ResourceRefs.of(agencyId, flowId, version, DataStructureRef::new); + public static DataStructureRef of(@Nullable String agency, @Nonnull String id, @Nullable String version) throws IllegalArgumentException { + return ResourceRefs.of(agency, id, version, DataStructureRef::new); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Dataflow.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Dataflow.java index 3eea97d5..58e3e2a0 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Dataflow.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Dataflow.java @@ -23,8 +23,11 @@ @lombok.Value(staticConstructor = "of") public class Dataflow implements HasLabel { - DataflowRef flowRef; - DataStructureRef dataStructureRef; + @lombok.NonNull + DataflowRef ref; + + @lombok.NonNull + DataStructureRef structureRef; @lombok.NonNull String label; diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java index 8d1a7fe6..e00ca7d4 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java @@ -40,7 +40,7 @@ public final class DataflowRef implements ResourceRef { @lombok.NonNull - private String agencyId; + private String agency; @lombok.NonNull private String id; @@ -49,7 +49,7 @@ public final class DataflowRef implements ResourceRef { private String version; public boolean contains(@Nonnull DataflowRef that) { - return (this.agencyId.equals(ALL_AGENCIES) || this.agencyId.equals(that.agencyId)) + return (this.agency.equals(ALL_AGENCIES) || this.agency.equals(that.agency)) && (this.id.equals(that.id)) && (this.version.equals(LATEST_VERSION) || this.version.equals(that.version)); } @@ -65,7 +65,7 @@ public static DataflowRef parse(@Nonnull String input) throws IllegalArgumentExc } @Nonnull - public static DataflowRef of(@Nullable String agencyId, @Nonnull String flowId, @Nullable String version) throws IllegalArgumentException { - return ResourceRefs.of(agencyId, flowId, version, DataflowRef::new); + public static DataflowRef of(@Nullable String agency, @Nonnull String id, @Nullable String version) throws IllegalArgumentException { + return ResourceRefs.of(agency, id, version, DataflowRef::new); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java index 703ded03..561041bd 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/ResourceRef.java @@ -29,7 +29,7 @@ public interface ResourceRef { public static final String LATEST_VERSION = "latest"; @Nonnull - String getAgencyId(); + String getAgency(); @Nonnull String getId(); diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java index 0260e4ba..913e31da 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java @@ -31,19 +31,19 @@ public interface SdmxConnection extends Closeable { @Nonnull - Set getDataflows() throws IOException; + Set getFlows() throws IOException; @Nonnull - Dataflow getDataflow(@Nonnull DataflowRef flowRef) throws IOException; + Dataflow getFlow(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataStructure getDataStructure(@Nonnull DataflowRef flowRef) throws IOException; + DataStructure getStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + DataCursor getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; @Nonnull - Stream getDataStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + Stream getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java index 6aaac08d..5eee7637 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/Series.java @@ -31,7 +31,7 @@ public class Series { Key key; @lombok.NonNull - Frequency frequency; + Frequency freq; @lombok.NonNull @lombok.Singular(value = "obs") diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java b/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java index 6020384a..b29fa68a 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/internal/util/ResourceRefs.java @@ -41,7 +41,7 @@ private static String nullOrEmptyToDefault(@Nullable String input, @Nonnull Stri @Nonnull public static String toString(ResourceRef ref) { - return ref.getAgencyId() + "," + ref.getId() + "," + ref.getVersion(); + return ref.getAgency() + "," + ref.getId() + "," + ref.getVersion(); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java index 05ffa5de..d1c63309 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java @@ -43,21 +43,22 @@ public void testParse() { } @Test + @SuppressWarnings("null") public void testValueOf() { assertThat(DataflowRef.of(null, "", null)) - .extracting(DataflowRef::getAgencyId, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) + .extracting(DataflowRef::getAgency, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) .containsExactly(ALL_AGENCIES, "", LATEST_VERSION, "all,,latest"); assertThat(DataflowRef.of("", "hello", null)) - .extracting(DataflowRef::getAgencyId, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) + .extracting(DataflowRef::getAgency, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) .containsExactly(ALL_AGENCIES, "hello", LATEST_VERSION, "all,hello,latest"); assertThat(DataflowRef.of("world", "hello", null)) - .extracting(DataflowRef::getAgencyId, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) + .extracting(DataflowRef::getAgency, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) .containsExactly("world", "hello", LATEST_VERSION, "world,hello,latest"); assertThat(DataflowRef.of("world", "hello", "123")) - .extracting(DataflowRef::getAgencyId, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) + .extracting(DataflowRef::getAgency, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) .containsExactly("world", "hello", "123", "world,hello,123"); assertThatThrownBy(() -> DataflowRef.of(null, "world,hello", null)).isInstanceOf(IllegalArgumentException.class); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 011363f2..05964756 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -82,7 +82,7 @@ private static List copyOfKeys(DataCursor cursor) throws IOException { List result = new ArrayList<>(); Series.Builder series = Series.builder(); while (cursor.nextSeries()) { - result.add(series.key(cursor.getSeriesKey()).frequency(cursor.getSeriesFrequency()).meta(cursor.getSeriesAttributes()).build()); + result.add(series.key(cursor.getSeriesKey()).freq(cursor.getSeriesFrequency()).meta(cursor.getSeriesAttributes()).build()); } return result; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 55eed361..a73a3496 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -64,13 +64,13 @@ class FileSdmxConnection implements SdmxConnection { } @Override - final public Set getDataflows() throws IOException { + final public Set getFlows() throws IOException { checkState(); return Collections.singleton(dataflow); } @Override - final public Dataflow getDataflow(DataflowRef flowRef) throws IOException { + final public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); Objects.requireNonNull(flowRef); checkFlowRef(flowRef); @@ -78,7 +78,7 @@ final public Dataflow getDataflow(DataflowRef flowRef) throws IOException { } @Override - final public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { + final public DataStructure getStructure(DataflowRef flowRef) throws IOException { checkState(); Objects.requireNonNull(flowRef); checkFlowRef(flowRef); @@ -86,7 +86,7 @@ final public DataStructure getDataStructure(DataflowRef flowRef) throws IOExcept } @Override - final public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { + final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); Objects.requireNonNull(flowRef); Objects.requireNonNull(query); @@ -95,8 +95,8 @@ final public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) thro } @Override - public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getDataCursor(flowRef, query)); + public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getCursor(flowRef, query)); } @Override @@ -139,7 +139,7 @@ private XMLStream getDataSupplier(SdmxDecoder.DataType o, DataStruct } private void checkFlowRef(DataflowRef flowRef) throws IOException { - if (!this.dataflow.getFlowRef().contains(flowRef)) { + if (!this.dataflow.getRef().contains(flowRef)) { throw new IOException("Invalid flowref '" + flowRef + "'"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index ab0c14ae..ac3fe702 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -50,12 +50,12 @@ public void testCompactData21() throws IOException { FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); - assertThat(conn.getDataflows()).hasSize(1); - assertThat(conn.getDataStructure(file.getDataflowRef()).getDimensions()).hasSize(7); + assertThat(conn.getFlows()).hasSize(1); + assertThat(conn.getStructure(file.getDataflowRef()).getDimensions()).hasSize(7); Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getDataCursor(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getCursor(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index d24f71fc..53240f90 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -52,37 +52,37 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getDataCursor(ref, DataQuery.of(Key.ALL, false)), "getDataCursor(DataFlowRef, DataQuery)"); - assertState(s, supplier, o -> o.getDataStream(ref, DataQuery.of(Key.ALL, false)), "getDataStream(DataFlowRef, DataQuery)"); - assertState(s, supplier, o -> o.getDataStructure(ref), "getDataStructure(DataFlowRef)"); - assertState(s, supplier, o -> o.getDataflow(ref), "getDataflow(DataFlowRef)"); - assertState(s, supplier, SdmxConnection::getDataflows, "getDataflows()"); + assertState(s, supplier, o -> o.getCursor(ref, DataQuery.of(Key.ALL, false)), "getCursor(DataFlowRef, DataQuery)"); + assertState(s, supplier, o -> o.getStream(ref, DataQuery.of(Key.ALL, false)), "getStream(DataFlowRef, DataQuery)"); + assertState(s, supplier, o -> o.getStructure(ref), "getStructure(DataFlowRef)"); + assertState(s, supplier, o -> o.getFlow(ref), "getFlow(DataFlowRef)"); + assertState(s, supplier, SdmxConnection::getFlows, "getFlows()"); } @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getDataCursor(null, DataQuery.of(Key.ALL, false))) - .as("Expecting 'getDataCursor(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getCursor(null, DataQuery.of(Key.ALL, false))) + .as("Expecting 'getCursor(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getDataCursor(ref, null)) - .as("Expecting 'getDataCursor(DataFlowRef, DataQuery)' to raise NPE when called with null query") + s.assertThatThrownBy(() -> conn.getCursor(ref, null)) + .as("Expecting 'getCursor(DataFlowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getDataStream(null, DataQuery.of(Key.ALL, false))) - .as("Expecting 'getDataStream(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getStream(null, DataQuery.of(Key.ALL, false))) + .as("Expecting 'getStream(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getDataStream(ref, null)) - .as("Expecting 'getDataStream(DataFlowRef, DataQuery)' to raise NPE when called with null query") + s.assertThatThrownBy(() -> conn.getStream(ref, null)) + .as("Expecting 'getStream(DataFlowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getDataStructure(null)) - .as("Expecting 'getDataStructure(DataFlowRef)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getStructure(null)) + .as("Expecting 'getStructure(DataFlowRef)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getDataflow(null)) - .as("Expecting 'getDataflow(DataFlowRef)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getFlow(null)) + .as("Expecting 'getFlow(DataFlowRef)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index a9742489..90ca5379 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -121,34 +121,34 @@ private RepoConnection(SdmxRepository repo) { } @Override - public Set getDataflows() throws IOException { + public Set getFlows() throws IOException { checkState(); return repo.getDataflows(); } @Override - public Dataflow getDataflow(DataflowRef flowRef) throws IOException { + public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); Objects.requireNonNull(flowRef); return repo.getDataflows() .stream() - .filter(o -> flowRef.equals(o.getFlowRef())) + .filter(o -> flowRef.equals(o.getRef())) .findFirst() .orElseThrow(() -> new IOException("Dataflow not found")); } @Override - public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { - Dataflow flow = getDataflow(flowRef); + public DataStructure getStructure(DataflowRef flowRef) throws IOException { + Dataflow flow = getFlow(flowRef); return repo.getDataStructures() .stream() - .filter(o -> flow.getDataStructureRef().equals(o.getRef())) + .filter(o -> flow.getStructureRef().equals(o.getRef())) .findFirst() .orElseThrow(() -> new IOException("DataStructure not found")); } @Override - public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { + public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); DataCursor result = repo.getCursor(flowRef, query); if (result != null) { @@ -158,7 +158,7 @@ public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOE } @Override - public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { + public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); Stream result = repo.getStream(flowRef, query); if (result != null) { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index 4ba7b32b..b95fca10 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -76,7 +76,7 @@ private Stream getDataStream(DataCursor cursor) { private Series getSeries(Series.Builder builder, DataCursor cursor) throws IOException { builder.key(cursor.getSeriesKey()) - .frequency(cursor.getSeriesFrequency()) + .freq(cursor.getSeriesFrequency()) .clearMeta() .clearObs() .meta(cursor.getSeriesAttributes()); @@ -132,7 +132,7 @@ public Key getSeriesKey() throws IOException { @Override public Frequency getSeriesFrequency() throws IOException { checkSeriesState(); - return col.get(i).getFrequency(); + return col.get(i).getFreq(); } @Override diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index fc65ed40..2774dd7a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -51,6 +51,6 @@ public void testBuilder() { } private final DataflowRef xyz = DataflowRef.parse("XYZ"); - private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); + private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); private final SdmxRepository repo = SdmxRepository.builder().name("test").data(xyz, series).build(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 304d0091..d6a16751 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -81,5 +81,5 @@ public void testCopyOf() throws IOException, XMLStreamException { } } - private final Series series = Series.builder().key(Key.of("BE")).frequency(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); + private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java index 3f0fb777..dc57430e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java @@ -65,27 +65,27 @@ interface Resource { } @Override - public Set getDataflows() throws IOException { + public Set getFlows() throws IOException { checkState(); return resource.loadDataFlowsById().values().stream() - .map(Util::toDataflow) + .map(Util::toFlow) .collect(Collectors.toSet()); } @Override - public Dataflow getDataflow(DataflowRef flowRef) throws IOException { + public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); - return Util.toDataflow(resource.loadDataflow(flowRef)); + return Util.toFlow(resource.loadDataflow(flowRef)); } @Override - public DataStructure getDataStructure(DataflowRef flowRef) throws IOException { + public DataStructure getStructure(DataflowRef flowRef) throws IOException { checkState(); - return Util.toDataStructure(resource.loadDataStructure(flowRef)); + return Util.toStructure(resource.loadDataStructure(flowRef)); } @Override - public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOException { + public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); if (serieskeysonly && !isSeriesKeysOnlySupported()) { @@ -95,8 +95,8 @@ public DataCursor getDataCursor(DataflowRef flowRef, DataQuery query) throws IOE } @Override - public Stream getDataStream(DataflowRef flowRef, DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getDataCursor(flowRef, query)); + public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getCursor(flowRef, query)); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java index 49b54782..5e1156c7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java @@ -65,7 +65,7 @@ public it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) t it.bancaditalia.oss.sdmx.api.Dataflow result; try { - result = client.getDataflow(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()); + result = client.getDataflow(flowRef.getId(), flowRef.getAgency(), flowRef.getVersion()); } catch (SdmxException ex) { throw expected(ex, "Failed to get details from dataset '%s'", flowRef); } catch (RuntimeException ex) { @@ -135,7 +135,7 @@ public boolean isSeriesKeysOnlySupported() { private it.bancaditalia.oss.sdmx.api.DSDIdentifier loadDsdIdentifier(DataflowRef flowRef) throws IOException { return client instanceof DotStat - ? new DSDIdentifier(flowRef.getId(), flowRef.getAgencyId(), flowRef.getVersion()) + ? new DSDIdentifier(flowRef.getId(), flowRef.getAgency(), flowRef.getVersion()) : loadDataflow(flowRef).getDsdIdentifier(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index b5b874b3..7378ad11 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -34,11 +34,11 @@ @lombok.experimental.UtilityClass public class Util { - public be.nbb.sdmx.facade.Dataflow toDataflow(Dataflow dataflow) { - return be.nbb.sdmx.facade.Dataflow.of(DataflowRef.parse(dataflow.getFullIdentifier()), toDataStructureRef(dataflow.getDsdIdentifier()), dataflow.getDescription()); + public be.nbb.sdmx.facade.Dataflow toFlow(Dataflow flow) { + return be.nbb.sdmx.facade.Dataflow.of(DataflowRef.parse(flow.getFullIdentifier()), toStructureRef(flow.getDsdIdentifier()), flow.getDescription()); } - public be.nbb.sdmx.facade.DataStructureRef toDataStructureRef(DSDIdentifier input) { + public be.nbb.sdmx.facade.DataStructureRef toStructureRef(DSDIdentifier input) { return DataStructureRef.of(input.getAgency(), input.getId(), input.getVersion()); } @@ -54,13 +54,13 @@ public be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { return result.build(); } - public DataStructure toDataStructure(DataFlowStructure dfs) { + public DataStructure toStructure(DataFlowStructure dsd) { DataStructure.Builder result = DataStructure.builder() - .ref(DataStructureRef.of(dfs.getAgency(), dfs.getId(), dfs.getVersion())) - .label(getNonNullName(dfs)) - .timeDimensionId(dfs.getTimeDimension()) - .primaryMeasureId(dfs.getMeasure()); - dfs.getDimensions().forEach(o -> result.dimension(toDimension(o))); + .ref(DataStructureRef.of(dsd.getAgency(), dsd.getId(), dsd.getVersion())) + .label(getNonNullName(dsd)) + .timeDimensionId(dsd.getTimeDimension()) + .primaryMeasureId(dsd.getMeasure()); + dsd.getDimensions().forEach(o -> result.dimension(toDimension(o))); return result.build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 71e20d68..25375287 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -92,7 +92,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index b24add24..5eba69e4 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -205,7 +205,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toDataStructure(dsd)).get(new XMLEventStreamReader(r))) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).get(new XMLEventStreamReader(r))) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index a57b361a..1ce6d815 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -60,8 +60,8 @@ public SdmxRepository nbb() throws IOException { DataflowRef ref = firstOf(flows); return SdmxRepository.builder() - .dataStructures(structs.stream().map(Util::toDataStructure).collect(Collectors.toList())) - .dataflows(flows.stream().map(Util::toDataflow).collect(Collectors.toList())) + .dataStructures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) + .dataflows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("NBB") .seriesKeysOnlySupported(false) @@ -79,8 +79,8 @@ public SdmxRepository ecb() throws IOException { DataflowRef ref = firstOf(flows); return SdmxRepository.builder() - .dataStructures(structs.stream().map(Util::toDataStructure).collect(Collectors.toList())) - .dataflows(flows.stream().map(Util::toDataflow).collect(Collectors.toList())) + .dataStructures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) + .dataflows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("ECB") .seriesKeysOnlySupported(true) @@ -88,7 +88,7 @@ public SdmxRepository ecb() throws IOException { } DataflowRef firstOf(List flows) { - return flows.stream().map(o -> Util.toDataflow(o).getFlowRef()).findFirst().get(); + return flows.stream().map(o -> Util.toFlow(o).getRef()).findFirst().get(); } List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { @@ -103,7 +103,7 @@ List flow20(ByteSource xml, LanguagePriorityList l) throws IOException List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data20(XMLInputFactory.newFactory(), xml, Util.toDataStructure(dsd)) + return FacadeResource.data20(XMLInputFactory.newFactory(), xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -119,7 +119,7 @@ List flow21(ByteSource xml, LanguagePriorityList l) throws IOException List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data21(XMLInputFactory.newFactory(), xml, Util.toDataStructure(dsd)) + return FacadeResource.data21(XMLInputFactory.newFactory(), xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -127,13 +127,13 @@ List data21(ByteSource xml, DataFlowStructure dsd, LanguageP PortableTimeSeries toPortableTimeSeries(Series o, List dims) { PortableTimeSeries result = new PortableTimeSeries(); - result.setFrequency(String.valueOf(formatByStandardFreq(o.getFrequency()))); + result.setFrequency(String.valueOf(formatByStandardFreq(o.getFreq()))); o.getMeta().forEach(result::addAttribute); Key key = o.getKey(); for (int i = 0; i < key.size(); i++) { result.addDimension(dims.get(i).getId(), key.get(i)); } - o.getObs().forEach(x -> result.addObservation(valueToString(x.getValue()), periodToString(o.getFrequency(), x.getPeriod()), null)); + o.getObs().forEach(x -> result.addObservation(valueToString(x.getValue()), periodToString(o.getFreq(), x.getPeriod()), null)); return result; } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index 3381cb1f..a083d287 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -106,7 +106,7 @@ private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorit // FIXME: no facade impl yet return ConnectorsResource.parse(xml, Util.fromLanguages(l), new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()) .stream() - .map(Util::toDataflow) + .map(Util::toFlow) .collect(Collectors.toList()); } @@ -117,7 +117,7 @@ List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws } private Dataflow asDataflow(DataStructure o) { - DataflowRef ref = DataflowRef.of(o.getRef().getAgencyId(), o.getRef().getId(), o.getRef().getVersion()); + DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); return Dataflow.of(ref, o.getRef(), o.getLabel()); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index faa9234d..c7dae274 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -52,7 +52,7 @@ public static void beforeClass() throws IOException { public void test() throws IOException { assertThat(SeriesSupport.asStream(() -> new PortableTimeSeriesCursor(DATA, ObsParser.standard()))) .hasSize(120) - .allMatch(o -> o.getFrequency().equals(Frequency.ANNUAL)) + .allMatch(o -> o.getFreq().equals(Frequency.ANNUAL)) .element(0) .satisfies(o -> { assertThat(o.getKey()).isEqualTo(Key.parse("A.DEU.1.0.319.0.UBLGE")); From 244a7fe70b74b0ae0946f4e5c7ad203c974c0e2b Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 27 Oct 2017 11:22:55 +0200 Subject: [PATCH 21/68] Improved use of api. --- .../main/java/be/nbb/demetra/dotstat/DotStatAccessor.java | 5 +++-- .../src/main/java/internal/sdmx/SdmxCubeAccessor.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java index 9ee1099f..f8a51079 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java @@ -133,8 +133,9 @@ private static DbSeries getSeriesWithData(SdmxConnection conn, DataflowRef flowR } private static List getChildren(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { - Converter converter = getConverter(conn.getStructure(flowRef), ref); - int dimensionPosition = dimensionById(conn.getStructure(flowRef)).get(ref.getColumn(ref.getLevel())).getPosition(); + DataStructure dsd = conn.getStructure(flowRef); + Converter converter = getConverter(dsd, ref); + int dimensionPosition = dimensionById(dsd).get(ref.getColumn(ref.getLevel())).getPosition(); return SdmxQueryUtil.getChildren(conn, flowRef, converter.convert(ref), dimensionPosition); } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index ec4c56d7..0e1e7059 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -178,8 +178,9 @@ private static TsCursor getSeriesWithDataCursor(SdmxConnection conn, Dat } private static IteratorWithIO getChildren(SdmxConnection conn, DataflowRef flowRef, CubeId ref) throws IOException { - Converter converter = getConverter(conn.getStructure(flowRef), ref); - int dimensionPosition = dimensionById(conn.getStructure(flowRef)).get(ref.getDimensionId(ref.getLevel())).getPosition(); + DataStructure dsd = conn.getStructure(flowRef); + Converter converter = getConverter(dsd, ref); + int dimensionPosition = dimensionById(dsd).get(ref.getDimensionId(ref.getLevel())).getPosition(); List children = SdmxQueryUtil.getChildren(conn, flowRef, converter.convert(ref), dimensionPosition); return IteratorWithIO.from(children.iterator()).transform(ref::child); } From 76d3826f24a341fe82f1fa38f7094bef48550353 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 30 Oct 2017 15:07:34 +0100 Subject: [PATCH 22/68] Updated WB entry point to enforce use of https. --- .../src/main/java/internal/connectors/drivers/Sdmx21Driver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 5eba69e4..481b0206 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -101,7 +101,7 @@ public List getDefaultEntryPoints() { result.add(b.clear() .name("WB") .description("World Bank") - .endpoint("http://api.worldbank.org/v2/sdmx/rest") + .endpoint("https://api.worldbank.org/v2/sdmx/rest") .supportsCompression(true) // .seriesKeysOnlySupported(true) .build()); From 675e44427ebf9d11ed08dfd98bebdf70811070c8 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 31 Oct 2017 09:47:05 +0100 Subject: [PATCH 23/68] Fixed insee missing codes. --- .../connectors/drivers/InseeDriver.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 25375287..2a6675fe 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -41,6 +41,9 @@ import internal.connectors.Util; import internal.org.springframework.util.xml.XMLEventStreamReader; import internal.util.drivers.InseeDataFactory; +import it.bancaditalia.oss.sdmx.api.Codelist; +import it.bancaditalia.oss.sdmx.api.DSDIdentifier; +import it.bancaditalia.oss.sdmx.api.Dimension; import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; @@ -72,6 +75,13 @@ private InseeClient(URI endpoint, LanguagePriorityList langs) { this.dataFactory = new InseeDataFactory(); } + @Override + public DataFlowStructure getDataFlowStructure(DSDIdentifier dsd, boolean full) throws SdmxException { + DataFlowStructure result = super.getDataFlowStructure(dsd, full); + fixMissingCodes(result); + return result; + } + @Override public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy @@ -83,6 +93,15 @@ public boolean isSeriesKeysOnlySupported() { return true; } + private void fixMissingCodes(DataFlowStructure dsd) throws SdmxException { + for (Dimension d : dsd.getDimensions()) { + Codelist freq = d.getCodeList(); + if (freq.getCodes().isEmpty()) { + freq.setCodes(super.getCodes(freq.getId(), freq.getAgency(), freq.getVersion())); + } + } + } + private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { return runQuery( getCompactData21Parser(dsd), From feb4267f00ba19dd695bf3a6bc86d85f79eaf96d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 31 Oct 2017 10:08:40 +0100 Subject: [PATCH 24/68] Code cleanup. --- .../java/be/nbb/sdmx/facade/util/SdmxFix.java | 19 +++++++++---------- .../xml/stream/XMLStreamStructure20.java | 6 ++++-- .../xml/stream/XMLStreamStructure21.java | 6 ++++-- .../main/java/internal/connectors/Util.java | 15 +++------------ .../connectors/drivers/InseeDriver.java | 6 ++++-- 5 files changed, 24 insertions(+), 28 deletions(-) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java index 5d0efb49..a4bce4be 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java @@ -16,20 +16,19 @@ */ package be.nbb.sdmx.facade.util; -import be.nbb.sdmx.facade.Dimension; -import java.util.Collections; -import java.util.Map; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; /** * * @author Philippe Charles */ -@lombok.experimental.UtilityClass -public class SdmxFix { +@Retention(RetentionPolicy.SOURCE) +@Documented +public @interface SdmxFix { - public void codes(@Nonnull Dimension.Builder dim, @Nullable Map codes) { - dim.codes(codes != null ? codes : Collections.emptyMap()); - } + String id(); + + String cause(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index 8d817714..a9fb1488 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -20,7 +20,6 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.util.SdmxFix; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -201,7 +200,10 @@ private void parseDimension(XMLStreamReader reader, DataStructure.Builder ds, Fu String conceptName = toConceptName.apply(id); result.label(conceptName != null ? conceptName : id); - SdmxFix.codes(result, toCodes.apply(codelist)); + Map codes = toCodes.apply(codelist); + if (codes != null) { + result.codes(codes); + } ds.dimension(result.build()); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index b759323f..4228e549 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -20,7 +20,6 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.util.SdmxFix; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -244,7 +243,10 @@ private void parseLocalRepresentation(XMLStreamReader reader, Dimension.Builder String id = reader.getAttributeValue(null, ID_ATTR); check(id != null, reader, "Missing Ref id"); - SdmxFix.codes(dimension, toCodes.apply(id)); + Map codes = toCodes.apply(id); + if (codes != null) { + dimension.codes(codes); + } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 7378ad11..77cc89bd 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -20,12 +20,10 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.util.Property.BoolProperty; -import be.nbb.sdmx.facade.util.SdmxFix; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.api.Dimension; -import javax.annotation.Nonnull; /** * @@ -45,11 +43,11 @@ public be.nbb.sdmx.facade.DataStructureRef toStructureRef(DSDIdentifier input) { public be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { be.nbb.sdmx.facade.Dimension.Builder result = be.nbb.sdmx.facade.Dimension.builder() .id(o.getId()) - .position(o.getPosition()); + .position(o.getPosition()) + .codes(o.getCodeList().getCodes()); String name = o.getName(); result.label(name != null ? name : o.getId()); - SdmxFix.codes(result, o.getCodeList().getCodes()); return result.build(); } @@ -57,7 +55,7 @@ public be.nbb.sdmx.facade.Dimension toDimension(Dimension o) { public DataStructure toStructure(DataFlowStructure dsd) { DataStructure.Builder result = DataStructure.builder() .ref(DataStructureRef.of(dsd.getAgency(), dsd.getId(), dsd.getVersion())) - .label(getNonNullName(dsd)) + .label(dsd.getName()) .timeDimensionId(dsd.getTimeDimension()) .primaryMeasureId(dsd.getMeasure()); dsd.getDimensions().forEach(o -> result.dimension(toDimension(o))); @@ -68,13 +66,6 @@ public it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.s return it.bancaditalia.oss.sdmx.util.LanguagePriorityList.parse(l.toString()); } - @Nonnull - private String getNonNullName(DataFlowStructure dfs) { - // FIXME: PR parsing code for name of data structure v2.1 in connectors - String result = dfs.getName(); - return result != null ? result : dfs.getId(); - } - public static final BoolProperty SUPPORTS_COMPRESSION = new BoolProperty("supportsCompression"); public static final BoolProperty NEEDS_CREDENTIALS = new BoolProperty("needsCredentials"); public static final BoolProperty NEEDS_URL_ENCODING = new BoolProperty("needsURLEncoding"); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 2a6675fe..ddbd2b44 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.util.SdmxFix; import be.nbb.sdmx.facade.util.SdmxMediaType; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; @@ -59,14 +60,15 @@ public final class InseeDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l)); + @SdmxFix(id = "INSEE#1", cause = "Fallback to http due to some servers that use root certificate unknown to jdk'") @Override public Collection getDefaultEntryPoints() { return ConnectorsDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); } - // private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { + @SdmxFix(id = "INSEE#2", cause = "Does not follow sdmx standard codes") private final InseeDataFactory dataFactory; private InseeClient(URI endpoint, LanguagePriorityList langs) { @@ -93,6 +95,7 @@ public boolean isSeriesKeysOnlySupported() { return true; } + @SdmxFix(id = "INSEE#3", cause = "Some codes are missing in dsd even when requested with 'references=children'") private void fixMissingCodes(DataFlowStructure dsd) throws SdmxException { for (Dimension d : dsd.getDimensions()) { Codelist freq = d.getCodeList(); @@ -119,5 +122,4 @@ private Parser> getCompactData21Parser(DataFlowStructure dsd) { }; } } - // } From db1177c09b35ea63de14e680cd85c88bcef254c5 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 31 Oct 2017 17:15:31 +0100 Subject: [PATCH 25/68] Fixed out-of-memory issue in SdmxFileProvider. --- .../demetra/sdmx/file/SdmxFileProvider.java | 2 +- .../be/nbb/sdmx/facade/file/SdmxFile.java | 46 ++-------- .../file/CachedFileSdmxConnection.java | 12 +-- .../main/java/internal/file/SdmxFileUtil.java | 83 +++++++++++++++++++ .../sdmx/facade/file/SdmxFileManagerTest.java | 2 +- .../be/nbb/sdmx/facade/file/SdmxFileTest.java | 75 +++++++++++++++++ .../internal/file/FileSdmxConnectionTest.java | 2 +- .../nbb/sdmx/facade/util/SeriesSupport.java | 14 ++++ .../sdmx/facade/util/SeriesSupportTest.java | 29 ++++++- 9 files changed, 209 insertions(+), 56 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java create mode 100644 sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index b77a896f..a8812326 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -175,7 +175,7 @@ private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorit } private static SdmxFile getFile(HasFilePaths paths, File data, File structure) throws FileNotFoundException { - return new SdmxFile(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); + return SdmxFile.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); } } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java index 7e2774ba..6063711a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java @@ -17,26 +17,22 @@ package be.nbb.sdmx.facade.file; import be.nbb.sdmx.facade.DataflowRef; +import internal.file.SdmxFileUtil; import java.io.File; -import java.io.StringReader; -import java.io.StringWriter; import javax.annotation.Nonnull; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.stream.XMLStreamWriter; +import javax.annotation.Nullable; /** * * @author Philippe Charles */ -@lombok.Value +@lombok.Value(staticConstructor = "of") public class SdmxFile { @lombok.NonNull File data; + @Nullable File structure; @Nonnull @@ -46,41 +42,11 @@ public DataflowRef getDataflowRef() { @Override public String toString() { - try { - StringWriter stream = new StringWriter(); - XMLStreamWriter writer = OUTPUT.createXMLStreamWriter(stream); - writer.writeStartElement("sdmxFile"); - writer.writeAttribute("data", data.toString()); - if (structure != null) { - writer.writeAttribute("structure", structure.toString()); - } - writer.writeEndElement(); - writer.close(); - return stream.toString(); - } catch (XMLStreamException ex) { - throw new RuntimeException(ex); - } + return SdmxFileUtil.toXml(this); } @Nonnull public static SdmxFile parse(@Nonnull String input) throws IllegalArgumentException { - try { - XMLStreamReader reader = INPUT.createXMLStreamReader(new StringReader(input)); - String data = null; - String structure = null; - while (reader.hasNext()) { - if (reader.next() == XMLStreamReader.START_ELEMENT && reader.getLocalName().equals("sdmxFile")) { - data = reader.getAttributeValue(null, "data"); - structure = reader.getAttributeValue(null, "structure"); - } - } - reader.close(); - return new SdmxFile(data != null ? new File(data) : null, structure != null ? new File(structure) : null); - } catch (XMLStreamException ex) { - throw new IllegalArgumentException("Cannot parse SdmxFile", ex); - } + return SdmxFileUtil.fromXml(input); } - - private static final XMLOutputFactory OUTPUT = XMLOutputFactory.newInstance(); - private static final XMLInputFactory INPUT = XMLInputFactory.newInstance(); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 05964756..b04cd2cd 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -27,7 +27,6 @@ import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; import java.time.Clock; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -70,20 +69,11 @@ protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key k if (serieskeysonly) { List result = cache.get(loadDataKey); if (result == null) { - result = copyOfKeys(super.loadData(entry, flowRef, key, true)); + result = SeriesSupport.copyOfKeysAndMeta(super.loadData(entry, flowRef, key, true)); cache.put(loadDataKey, result); } return SeriesSupport.asCursor(result, key); } return super.loadData(entry, flowRef, key, serieskeysonly); } - - private static List copyOfKeys(DataCursor cursor) throws IOException { - List result = new ArrayList<>(); - Series.Builder series = Series.builder(); - while (cursor.nextSeries()) { - result.add(series.key(cursor.getSeriesKey()).freq(cursor.getSeriesFrequency()).meta(cursor.getSeriesAttributes()).build()); - } - return result; - } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java new file mode 100644 index 00000000..a3bec60e --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -0,0 +1,83 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.file; + +import be.nbb.sdmx.facade.file.SdmxFile; +import java.io.File; +import java.io.StringReader; +import java.io.StringWriter; +import javax.annotation.Nonnull; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class SdmxFileUtil { + + @Nonnull + public String toXml(@Nonnull SdmxFile file) { + StringWriter result = new StringWriter(); + try { + XMLStreamWriter xml = OUTPUT.createXMLStreamWriter(result); + xml.writeEmptyElement(ROOT_TAG); + xml.writeAttribute(DATA_ATTR, file.getData().toString()); + File structure = file.getStructure(); + if (structure != null) { + xml.writeAttribute(STRUCT_ATTR, structure.toString()); + } + xml.writeEndDocument(); + xml.close(); + } catch (XMLStreamException ex) { + throw new RuntimeException(ex); + } + return result.toString(); + } + + @Nonnull + public static SdmxFile fromXml(@Nonnull String input) throws IllegalArgumentException { + String data = null; + String structure = null; + try { + XMLStreamReader xml = INPUT.createXMLStreamReader(new StringReader(input)); + while (xml.hasNext()) { + if (xml.next() == XMLStreamReader.START_ELEMENT && xml.getLocalName().equals(ROOT_TAG)) { + data = xml.getAttributeValue(null, DATA_ATTR); + structure = xml.getAttributeValue(null, STRUCT_ATTR); + } + } + xml.close(); + } catch (XMLStreamException ex) { + throw new IllegalArgumentException("Cannot parse SdmxFile", ex); + } + if (data == null) { + throw new IllegalArgumentException("Cannot parse SdmxFile from '" + input + "'"); + } + return SdmxFile.of(new File(data), structure != null ? new File(structure) : null); + } + + private static final String ROOT_TAG = "file"; + private static final String DATA_ATTR = "data"; + private static final String STRUCT_ATTR = "structure"; + private static final XMLOutputFactory OUTPUT = XMLOutputFactory.newInstance(); + private static final XMLInputFactory INPUT = XMLInputFactory.newInstance(); +} diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index 4d7b245a..01ad51b4 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -38,7 +38,7 @@ public void testCompliance() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFile file = new SdmxFile(compact21, null); + SdmxFile file = SdmxFile.of(compact21, null); ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), file.toString(), "ko"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java new file mode 100644 index 00000000..e84f68b6 --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.file; + +import java.io.File; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxFileTest { + + @Test + @SuppressWarnings("null") + public void testFactory() { + assertThatThrownBy(() -> SdmxFile.of(null, null)).isInstanceOf(NullPointerException.class); + + assertThat(SdmxFile.of(data, null)) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", null); + + assertThat(SdmxFile.of(data, structure)) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", structure); + } + + @Test + public void testToString() { + assertThat(SdmxFile.of(data, structure).toString()) + .isEqualTo(""); + + assertThat(SdmxFile.of(data, null).toString()) + .isEqualTo(""); + } + + @Test + @SuppressWarnings("null") + public void testParse() { + assertThatThrownBy(() -> SdmxFile.parse(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxFile.parse("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFile.parse("")).isInstanceOf(IllegalArgumentException.class); + + assertThat(SdmxFile.parse("")) + .hasFieldOrPropertyWithValue("data", new File("")) + .hasFieldOrPropertyWithValue("structure", null); + + assertThat(SdmxFile.parse("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", null); + + assertThat(SdmxFile.parse("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", structure); + } + + private final File data = new File("a.xml"); + private final File structure = new File("b.xml"); +} diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index ac3fe702..62439e0e 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -46,7 +46,7 @@ public void testCompactData21() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFile file = new SdmxFile(compact21, null); + SdmxFile file = SdmxFile.of(compact21, null); FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index b95fca10..d3d463bd 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -56,6 +56,20 @@ public List copyOf(@Nonnull DataCursor cursor) throws IOException { return result; } + @Nonnull + public static List copyOfKeysAndMeta(@Nonnull DataCursor cursor) throws IOException { + if (!cursor.nextSeries()) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(); + Series.Builder builder = Series.builder(); + do { + result.add(builder.key(cursor.getSeriesKey()).freq(cursor.getSeriesFrequency()).clearMeta().meta(cursor.getSeriesAttributes()).build()); + } while (cursor.nextSeries()); + return result; + } + @Nonnull public DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { Objects.requireNonNull(list); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index d6a16751..5ccde929 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -76,8 +76,33 @@ public void testCopyOf() throws IOException, XMLStreamException { } List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); - try (DataCursor o = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { - assertThat(SeriesSupport.copyOf(o)).hasSize(120); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + assertThat(SeriesSupport.copyOf(c)).hasSize(120); + } + } + + @Test + @SuppressWarnings("null") + public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { + assertThatThrownBy(() -> SeriesSupport.copyOfKeysAndMeta(null)).isInstanceOf(NullPointerException.class); + + try (DataCursor c = NoOpCursor.noOp()) { + assertThat(SeriesSupport.copyOfKeysAndMeta(c)).isEmpty(); + } + + try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(series), Key.ALL)) { + assertThat(SeriesSupport.copyOfKeysAndMeta(c)) + .allMatch(o -> o.getObs().isEmpty()) + .hasSize(1) + .element(0) + .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); + } + + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + assertThat(SeriesSupport.copyOfKeysAndMeta(c)) + .allMatch(o -> o.getObs().isEmpty()) + .hasSize(120); } } From 25530e14a802ccf97465b30f447b414a57b14398 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 8 Nov 2017 16:16:29 +0100 Subject: [PATCH 26/68] Fixed missing meta when retrieving single ts. --- .../nbb/demetra/dotstat/DotStatAccessor.java | 4 +- .../java/internal/sdmx/SdmxCubeAccessor.java | 6 +- .../java/internal/sdmx/SdmxQueryUtil.java | 9 +- .../sdmx/file/SdmxFileProviderTest.java | 87 +++++++++++-------- .../java/internal/sdmx/SdmxQueryUtilTest.java | 38 ++++---- 5 files changed, 84 insertions(+), 60 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java index f8a51079..bb09f052 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatAccessor.java @@ -129,7 +129,9 @@ private static DbSeries getSeriesWithData(SdmxConnection conn, DataflowRef flowR Converter converter = getConverter(conn.getStructure(flowRef), ref); Key seriesKey = converter.convert(ref); - return new DbSeries(ref, SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey)); + try (TsCursor cursor = SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey, SdmxQueryUtil.NO_LABEL)) { + return new DbSeries(ref, cursor.nextSeries() ? cursor.getSeriesData() : SdmxQueryUtil.MISSING_DATA); + } } private static List getChildren(SdmxConnection conn, DataflowRef flowRef, DbSetId ref) throws IOException { diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index 0e1e7059..4d417a02 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -94,7 +94,7 @@ public TsCursor getAllSeriesWithData(CubeId ref) throws IOException { public TsCursor getSeriesWithData(CubeId ref) throws IOException { SdmxConnection conn = supplier.getConnection(source, languages); try { - return getSeriesWithDataCursor(conn, flowRef, ref).onClose(conn); + return getSeriesWithDataCursor(conn, flowRef, ref, labelAttribute).onClose(conn); } catch (IOException ex) { throw close(conn, ex); } catch (RuntimeException ex) { @@ -168,11 +168,11 @@ private static TsCursor getAllSeriesWithDataCursor(SdmxConnection conn, return cursor.transform(converter.reverse()::convert); } - private static TsCursor getSeriesWithDataCursor(SdmxConnection conn, DataflowRef flowRef, CubeId ref) throws IOException { + private static TsCursor getSeriesWithDataCursor(SdmxConnection conn, DataflowRef flowRef, CubeId ref, String labelAttribute) throws IOException { Converter converter = getConverter(conn.getStructure(flowRef), ref); Key seriesKey = converter.convert(ref); - TsCursor cursor = TsCursor.singleton(seriesKey, SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey)); + TsCursor cursor = SdmxQueryUtil.getSeriesWithData(conn, flowRef, seriesKey, labelAttribute); return cursor.transform(converter.reverse()::convert); } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index e5af72b0..544319e8 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -46,6 +46,7 @@ public class SdmxQueryUtil { public final String NO_LABEL = null; + public final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); @Nonnull public TsCursor getAllSeries(SdmxConnection conn, DataflowRef flowRef, Key ref, @Nullable String labelAttribute) throws IOException { @@ -62,10 +63,8 @@ public TsCursor getAllSeriesWithData(SdmxConnection conn, DataflowRef flowR } @Nonnull - public OptionalTsData getSeriesWithData(SdmxConnection conn, DataflowRef flowRef, Key ref) throws IOException { - try (TsCursor cursor = request(conn, flowRef, ref, NO_LABEL, false)) { - return cursor.nextSeries() ? cursor.getSeriesData() : MISSING_DATA; - } + public TsCursor getSeriesWithData(SdmxConnection conn, DataflowRef flowRef, Key ref, @Nullable String labelAttribute) throws IOException { + return request(conn, flowRef, ref, labelAttribute, false); } @Nonnull @@ -85,8 +84,6 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re } // - private final OptionalTsData MISSING_DATA = OptionalTsData.absent("No results matching the query"); - private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { return new SdmxDataAdapter(key, conn.getCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); } diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java index 7d6c4a49..cfd42794 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java @@ -20,9 +20,9 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; import ec.tss.TsCollectionInformation; +import ec.tss.TsInformation; import ec.tss.TsInformationType; import ec.tss.TsMoniker; -import static ec.tss.tsproviders.Assertions.assertThat; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.DataSource; import ec.tss.tsproviders.IDataSourceLoaderAssert; @@ -30,7 +30,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import org.assertj.core.api.Assertions; +import java.util.concurrent.atomic.AtomicReference; +import static org.assertj.core.api.Assertions.*; import org.junit.BeforeClass; import org.junit.Test; @@ -97,59 +98,75 @@ private static SdmxFileProvider getProvider() { @Test public void testContent() throws IOException { try (SdmxFileProvider p = new SdmxFileProvider()) { - Assertions.assertThat(newRequest(p, GENERIC20, STRUCT20)).satisfies(info -> { - Assertions.assertThat(p.get(info)).isTrue(); - Assertions.assertThat(info.items) + + AtomicReference single = new AtomicReference<>(); + assertThat(newColInfo(p, GENERIC20, STRUCT20)).satisfies(info -> { + assertThat(p.get(info)).isTrue(); + assertThat(info.items) .hasSize(1) .element(0) .satisfies(o -> { - Assertions.assertThat(o.name).isEqualTo("LOCSTL04.AUS.M"); - Assertions.assertThat(new HashMap(o.metaData)).hasSize(1).containsEntry(TIME_FORMAT_CONCEPT, "P1M"); - Assertions.assertThat(o.data).isNotNull(); - Assertions.assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("Monthly"); + assertThat(o.name).isEqualTo("LOCSTL04.AUS.M"); + assertThat(new HashMap(o.metaData)).hasSize(1).containsEntry(TIME_FORMAT_CONCEPT, "P1M"); + assertThat(o.data).isNotNull(); + assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("Monthly"); + single.set(o.moniker); }); }); - Assertions.assertThat(newRequest(p, GENERIC20, BLANK)).satisfies(info -> { - Assertions.assertThat(p.get(info)).isTrue(); - Assertions.assertThat(info.items).hasSize(1).element(0).satisfies(o -> Assertions.assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("M")); + p.clearCache(); + assertThat(new TsInformation("", single.get(), TsInformationType.All)).satisfies(o -> { + assertThat(p.get(o)).isTrue(); + assertThat(o.name).isEqualTo("LOCSTL04.AUS.M"); + assertThat(new HashMap(o.metaData)).hasSize(1).containsEntry(TIME_FORMAT_CONCEPT, "P1M"); + assertThat(o.data).isNotNull(); + assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("Monthly"); + single.set(o.moniker); + }); + + assertThat(newColInfo(p, GENERIC20, BLANK)).satisfies(info -> { + assertThat(p.get(info)).isTrue(); + assertThat(info.items).hasSize(1).element(0).satisfies(o -> assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("M")); }); - Assertions.assertThat(newRequest(p, GENERIC20, BLANK, "SUBJECT", "LOCATION", "FREQUENCY")).satisfies(info -> { - Assertions.assertThat(p.get(info)).isTrue(); - Assertions.assertThat(info.items).hasSize(1).element(0).satisfies(o -> Assertions.assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("M")); + assertThat(newColInfo(p, GENERIC20, BLANK, "SUBJECT", "LOCATION", "FREQUENCY")).satisfies(info -> { + assertThat(p.get(info)).isTrue(); + assertThat(info.items).hasSize(1).element(0).satisfies(o -> assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("M")); }); - Assertions.assertThat(newRequest(p, GENERIC20, BLANK, "LOCATION", "FREQUENCY", "SUBJECT")).satisfies(info -> { - Assertions.assertThat(p.get(info)).isTrue(); - Assertions.assertThat(info.items).hasSize(1).element(0).satisfies(o -> Assertions.assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("LOCSTL04")); + assertThat(newColInfo(p, GENERIC20, BLANK, "LOCATION", "FREQUENCY", "SUBJECT")).satisfies(info -> { + assertThat(p.get(info)).isTrue(); + assertThat(info.items).hasSize(1).element(0).satisfies(o -> assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("LOCSTL04")); }); - Assertions.assertThat(newRequest(p, GENERIC20, STRUCT20, "LOCATION", "FREQUENCY", "SUBJECT")).satisfies(info -> { - Assertions.assertThat(p.get(info)).isTrue(); - Assertions.assertThat(info.items).hasSize(1).element(0).satisfies(o -> Assertions.assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("Amplitude adjusted (CLI)")); + assertThat(newColInfo(p, GENERIC20, STRUCT20, "LOCATION", "FREQUENCY", "SUBJECT")).satisfies(info -> { + assertThat(p.get(info)).isTrue(); + assertThat(info.items).hasSize(1).element(0).satisfies(o -> assertThat(p.getDisplayNodeName(p.toDataSet(o.moniker))).isEqualTo("Amplitude adjusted (CLI)")); }); - Assertions.assertThat(newRequest(p, NO_XML, STRUCT20)).satisfies(info -> { - Assertions.assertThat(p.get(info)).isFalse(); - Assertions.assertThat(info.invalidDataCause).contains("XMLStreamException"); + assertThat(newColInfo(p, NO_XML, STRUCT20)).satisfies(info -> { + assertThat(p.get(info)).isFalse(); + assertThat(info.invalidDataCause).contains("XMLStreamException"); }); - Assertions.assertThat(newRequest(p, GENERIC20, NO_XML)).satisfies(info -> { - Assertions.assertThat(p.get(info)).isFalse(); - Assertions.assertThat(info.invalidDataCause).contains("XMLStreamException"); + assertThat(newColInfo(p, GENERIC20, NO_XML)).satisfies(info -> { + assertThat(p.get(info)).isFalse(); + assertThat(info.invalidDataCause).contains("XMLStreamException"); }); } } - private static TsCollectionInformation newRequest(SdmxFileProvider p, File data, File structure, String... dims) { - SdmxFileBean bean = new SdmxFileBean(); - bean.setFile(data); - bean.setStructureFile(structure); - bean.setDimensions(Arrays.asList(dims)); - - DataSource dataSource = p.encodeBean(bean); - Assertions.assertThat(p.open(dataSource)).isTrue(); + private static TsCollectionInformation newColInfo(SdmxFileProvider p, File data, File structure, String... dims) { + DataSource dataSource = p.encodeBean(newBean(data, structure, dims)); + assertThat(p.open(dataSource)).isTrue(); return new TsCollectionInformation(p.toMoniker(dataSource), TsInformationType.All); } + + private static SdmxFileBean newBean(File data, File structure, String... dims) { + SdmxFileBean result = new SdmxFileBean(); + result.setFile(data); + result.setStructureFile(structure); + result.setDimensions(Arrays.asList(dims)); + return result; + } } diff --git a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java index 51a5e965..21c0802d 100644 --- a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java +++ b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java @@ -113,14 +113,18 @@ public void testGetAllSeriesWithData20() throws Exception { public void testGetSeriesWithData20() throws Exception { SdmxConnection conn = nbb.asConnection(); - TsData o = getSeriesWithData(conn, nbbFlow, Key.of("LOCSTL04", "AUS", "M")).get(); - assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Monthly, 1966, 1)); - assertThat(o.getLastPeriod()).isEqualTo(new TsPeriod(TsFrequency.Monthly, 1970, 7)); - assertThat(o.getLength()).isEqualTo(55); - assertThat(o.getObsCount()).isEqualTo(54); - assertThat(o.isMissing(50)).isTrue(); // 1970-04 - assertThat(o.get(0)).isEqualTo(98.68823); - assertThat(o.get(54)).isEqualTo(101.1945); + try (TsCursor c = getSeriesWithData(conn, nbbFlow, Key.of("LOCSTL04", "AUS", "M"), SdmxQueryUtil.NO_LABEL)) { + assertThat(c.nextSeries()).isTrue(); + TsData o = c.getSeriesData().get(); + assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Monthly, 1966, 1)); + assertThat(o.getLastPeriod()).isEqualTo(new TsPeriod(TsFrequency.Monthly, 1970, 7)); + assertThat(o.getLength()).isEqualTo(55); + assertThat(o.getObsCount()).isEqualTo(54); + assertThat(o.isMissing(50)).isTrue(); // 1970-04 + assertThat(o.get(0)).isEqualTo(98.68823); + assertThat(o.get(54)).isEqualTo(101.1945); + assertThat(c.nextSeries()).isFalse(); + } } @Test @@ -226,13 +230,17 @@ public void testGetSeriesWithData21() throws Exception { Key key = Key.of("A", "DEU", "1", "0", "319", "0", "UBLGE"); - TsData o = getSeriesWithData(conn, ecbFlow, key).get(); - assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Yearly, 1991, 0)); - assertThat(o.getLastPeriod()).isEqualTo(new TsPeriod(TsFrequency.Yearly, 2015, 0)); - assertThat(o.getLength()).isEqualTo(25); - assertThat(o.getObsCount()).isEqualTo(25); - assertThat(o.get(0)).isEqualTo(-2.8574221); - assertThat(o.get(24)).isEqualTo(-0.1420473); + try (TsCursor c = getSeriesWithData(conn, ecbFlow, key, SdmxQueryUtil.NO_LABEL)) { + assertThat(c.nextSeries()).isTrue(); + TsData o = c.getSeriesData().get(); + assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Yearly, 1991, 0)); + assertThat(o.getLastPeriod()).isEqualTo(new TsPeriod(TsFrequency.Yearly, 2015, 0)); + assertThat(o.getLength()).isEqualTo(25); + assertThat(o.getObsCount()).isEqualTo(25); + assertThat(o.get(0)).isEqualTo(-2.8574221); + assertThat(o.get(24)).isEqualTo(-0.1420473); + assertThat(c.nextSeries()).isFalse(); + } } @Test From 40e8c9982c2ec5ee99ec9605e4871750c65726ba Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 09:18:54 +0100 Subject: [PATCH 27/68] Code cleanup. --- .../nbb/demetra/sdmx/HasSdmxProperties.java | 2 + .../demetra/sdmx/file/SdmxFileProvider.java | 47 +++--------- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 43 ++--------- .../internal/sdmx/SdmxPropertiesSupport.java | 76 +++++++++++++++++++ 4 files changed, 95 insertions(+), 73 deletions(-) create mode 100644 demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/HasSdmxProperties.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/HasSdmxProperties.java index 584ec985..52bae038 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/HasSdmxProperties.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/HasSdmxProperties.java @@ -20,11 +20,13 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; /** * * @author Philippe Charles */ +@ThreadSafe public interface HasSdmxProperties { @Nonnull diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index a8812326..ff22dff7 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -40,11 +40,11 @@ import ec.tss.tsproviders.utils.IParam; import ec.tstoolkit.utilities.GuavaCaches; import internal.sdmx.SdmxCubeItems; +import internal.sdmx.SdmxPropertiesSupport; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,8 +58,8 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private static final String NAME = "sdmx-file"; - private final AtomicReference connectionSupplier; - private final AtomicReference languages; + @lombok.experimental.Delegate + private final SdmxPropertiesSupport properties; @lombok.experimental.Delegate private final HasDataSourceMutableList mutableListSupport; @@ -80,18 +80,16 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private final ITsProvider tsSupport; public SdmxFileProvider() { - this.connectionSupplier = new AtomicReference<>(SdmxFileManager.of()); - this.languages = new AtomicReference<>(LanguagePriorityList.ANY); - Logger logger = LoggerFactory.getLogger(NAME); Cache cache = GuavaCaches.softValuesCache(); SdmxFileParam sdmxParam = new SdmxFileParam.V1(); + this.properties = SdmxPropertiesSupport.of(SdmxFileManager::of, cache::invalidateAll, () -> LanguagePriorityList.ANY, cache::invalidateAll); this.mutableListSupport = HasDataSourceMutableList.of(NAME, logger, cache::invalidate); this.monikerSupport = HasDataMoniker.usingUri(NAME); this.beanSupport = HasDataSourceBean.of(NAME, sdmxParam, sdmxParam.getVersion()); this.filePathSupport = HasFilePaths.of(cache::invalidateAll); - this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, connectionSupplier, languages, filePathSupport, sdmxParam)); + this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, properties, filePathSupport, sdmxParam)); this.tsSupport = CubeSupport.asTsProvider(NAME, logger, cubeSupport, monikerSupport, cache::invalidateAll); } @@ -105,38 +103,11 @@ public boolean accept(File pathname) { return pathname.getName().toLowerCase().endsWith(".xml"); } - @Override - public SdmxConnectionSupplier getConnectionSupplier() { - return connectionSupplier.get(); - } - - @Override - public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { - SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxFileManager.of())) { - clearCache(); - } - } - - @Override - public LanguagePriorityList getLanguages() { - return languages.get(); - } - - @Override - public void setLanguages(LanguagePriorityList languages) { - LanguagePriorityList old = this.languages.get(); - if (this.languages.compareAndSet(old, languages != null ? languages : LanguagePriorityList.ANY)) { - clearCache(); - } - } - @lombok.AllArgsConstructor private static final class SdmxCubeResource implements CubeSupport.Resource { private final Cache cache; - private final AtomicReference supplier; - private final AtomicReference languages; + private final HasSdmxProperties properties; private final HasFilePaths paths; private final SdmxFileParam param; @@ -152,12 +123,12 @@ public IParam getIdParam(DataSource dataSource) throws IOExcept private SdmxCubeItems get(DataSource dataSource) throws IOException { DataSourcePreconditions.checkProvider(NAME, dataSource); - return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(supplier.get(), languages.get(), paths, param, dataSource)); + return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties.getConnectionSupplier(), properties.getLanguages(), paths, param, dataSource)); } private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, HasFilePaths paths, SdmxFileParam param, DataSource dataSource) throws IOException { SdmxFileBean bean = param.get(dataSource); - SdmxFile file = getFile(paths, bean.getFile(), bean.getStructureFile()); + SdmxFile file = resolveFile(paths, bean.getFile(), bean.getStructureFile()); String source = file.toString(); DataflowRef flow = file.getDataflowRef(); @@ -174,7 +145,7 @@ private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorit return new SdmxCubeItems(accessor, idParam); } - private static SdmxFile getFile(HasFilePaths paths, File data, File structure) throws FileNotFoundException { + private static SdmxFile resolveFile(HasFilePaths paths, File data, File structure) throws FileNotFoundException { return SdmxFile.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); } } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index 87eceb8f..53e6ec11 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -38,10 +38,10 @@ import ec.tss.tsproviders.utils.IParam; import ec.tstoolkit.utilities.GuavaCaches; import internal.sdmx.SdmxCubeItems; +import internal.sdmx.SdmxPropertiesSupport; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import org.openide.util.lookup.ServiceProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,10 +56,11 @@ public final class SdmxWebProvider implements IDataSourceLoader, HasSdmxProperti public static final String NAME = "DOTSTAT"; - private final AtomicReference connectionSupplier; - private final AtomicReference languages; private final AtomicBoolean displayCodes; + @lombok.experimental.Delegate + private final SdmxPropertiesSupport properties; + @lombok.experimental.Delegate private final HasDataSourceMutableList mutableListSupport; @@ -76,18 +77,17 @@ public final class SdmxWebProvider implements IDataSourceLoader, HasSdmxProperti private final ITsProvider tsSupport; public SdmxWebProvider() { - this.connectionSupplier = new AtomicReference<>(SdmxWebManager.ofServiceLoader()); - this.languages = new AtomicReference<>(LanguagePriorityList.ANY); this.displayCodes = new AtomicBoolean(false); Cache cache = GuavaCaches.softValuesCache(); Logger logger = LoggerFactory.getLogger(NAME); SdmxWebParam beanParam = new SdmxWebParam.V1(); + this.properties = SdmxPropertiesSupport.of(SdmxWebManager::ofServiceLoader, cache::invalidateAll, () -> LanguagePriorityList.ANY, cache::invalidateAll); this.mutableListSupport = HasDataSourceMutableList.of(NAME, logger, cache::invalidate); this.monikerSupport = HasDataMoniker.usingUri(NAME); this.beanSupport = HasDataSourceBean.of(NAME, beanParam, beanParam.getVersion()); - this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, connectionSupplier, languages, beanParam)); + this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, properties, beanParam)); this.tsSupport = CubeSupport.asTsProvider(NAME, logger, cubeSupport, monikerSupport, cache::invalidateAll); } @@ -96,32 +96,6 @@ public String getDisplayName() { return "SDMX Web Services"; } - @Override - public SdmxConnectionSupplier getConnectionSupplier() { - return connectionSupplier.get(); - } - - @Override - public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { - SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxWebManager.ofServiceLoader())) { - clearCache(); - } - } - - @Override - public LanguagePriorityList getLanguages() { - return languages.get(); - } - - @Override - public void setLanguages(LanguagePriorityList languages) { - LanguagePriorityList old = this.languages.get(); - if (this.languages.compareAndSet(old, languages != null ? languages : LanguagePriorityList.ANY)) { - clearCache(); - } - } - public boolean isDisplayCodes() { return displayCodes.get(); } @@ -137,8 +111,7 @@ public void setDisplayCodes(boolean displayCodes) { private static final class SdmxCubeResource implements CubeSupport.Resource { private final Cache cache; - private final AtomicReference supplier; - private final AtomicReference languages; + private final SdmxPropertiesSupport properties; private final SdmxWebParam param; @Override @@ -153,7 +126,7 @@ public IParam getIdParam(DataSource dataSource) throws IOExcept private SdmxCubeItems get(DataSource dataSource) throws IOException { DataSourcePreconditions.checkProvider(NAME, dataSource); - return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(supplier.get(), languages.get(), param, dataSource)); + return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties.getConnectionSupplier(), properties.getLanguages(), param, dataSource)); } private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, SdmxWebParam param, DataSource dataSource) throws IllegalArgumentException, IOException { diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java new file mode 100644 index 00000000..6ef070cd --- /dev/null +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.sdmx; + +import be.nbb.demetra.sdmx.HasSdmxProperties; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import lombok.AccessLevel; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +public final class SdmxPropertiesSupport implements HasSdmxProperties { + + public static SdmxPropertiesSupport of(Supplier defaultSupplier, Runnable onSupplierChange, Supplier defaultLanguages, Runnable onLanguagesChange) { + return new SdmxPropertiesSupport( + defaultSupplier, + new AtomicReference<>(defaultSupplier.get()), + onSupplierChange, + defaultLanguages, + new AtomicReference<>(defaultLanguages.get()), + onLanguagesChange); + } + + private final Supplier defaultSupplier; + private final AtomicReference supplier; + private final Runnable onSupplierChange; + + private final Supplier defaultLanguages; + private final AtomicReference languages; + private final Runnable onLanguagesChange; + + @Override + public SdmxConnectionSupplier getConnectionSupplier() { + return supplier.get(); + } + + @Override + public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { + SdmxConnectionSupplier old = this.supplier.get(); + if (this.supplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : defaultSupplier.get())) { + onSupplierChange.run(); + } + } + + @Override + public LanguagePriorityList getLanguages() { + return languages.get(); + } + + @Override + public void setLanguages(LanguagePriorityList languages) { + LanguagePriorityList old = this.languages.get(); + if (this.languages.compareAndSet(old, languages != null ? languages : defaultLanguages.get())) { + onLanguagesChange.run(); + } + } +} From df526e60576344041a065daeb5606402439d1243 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 10:50:07 +0100 Subject: [PATCH 28/68] Refactored SdmwWebDriver to be more generic. --- .../java/be/nbb/sdmx/facade/web/SdmxWebManager.java | 8 ++++---- .../be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java | 6 ++---- .../connectors/ConnectorsDriverSupport.java | 13 +++++++------ .../be/nbb/sdmx/facade/web/SdmxWebManagerTest.java | 10 ++++------ 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 3664523b..950a0ee2 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -92,7 +92,7 @@ public SdmxConnection getConnection(@Nonnull SdmxWebEntryPoint entryPoint, @Nonn Objects.requireNonNull(languages); for (SdmxWebDriver o : drivers) { - if (tryAcceptURI(o, entryPoint)) { + if (tryAccepts(o, entryPoint)) { return tryConnect(o, entryPoint, languages); } } @@ -130,9 +130,9 @@ private static void updateEntryPointMap(ConcurrentMap list.forEach(o -> entryPointByName.put(o.getName(), o)); } - private static boolean tryAcceptURI(SdmxWebDriver driver, SdmxWebEntryPoint entryPoint) throws IOException { + private static boolean tryAccepts(SdmxWebDriver driver, SdmxWebEntryPoint entryPoint) throws IOException { try { - return driver.acceptsURI(entryPoint.getUri()); + return driver.accepts(entryPoint); } catch (RuntimeException ex) { log.log(Level.WARNING, "Unexpected exception while parsing URI", ex); return false; @@ -144,7 +144,7 @@ private static SdmxConnection tryConnect(SdmxWebDriver driver, SdmxWebEntryPoint SdmxConnection result; try { - result = driver.connect(entryPoint.getUri(), entryPoint.getProperties(), languages); + result = driver.connect(entryPoint, languages); } catch (RuntimeException ex) { log.log(Level.WARNING, "Unexpected exception while connecting", ex); throw new UnexpectedIOException(ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java index 7fbc0e7f..28a8024b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java @@ -20,9 +20,7 @@ import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import java.io.IOException; -import java.net.URI; import java.util.Collection; -import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; @@ -34,9 +32,9 @@ public interface SdmxWebDriver { @Nonnull - SdmxConnection connect(@Nonnull URI uri, @Nonnull Map properties, @Nonnull LanguagePriorityList languages) throws IOException; + SdmxConnection connect(@Nonnull SdmxWebEntryPoint entryPoint, @Nonnull LanguagePriorityList languages) throws IOException; - boolean acceptsURI(@Nonnull URI uri) throws IOException; + boolean accepts(@Nonnull SdmxWebEntryPoint entryPoint) throws IOException; @Nonnull Collection getDefaultEntryPoints(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java index 8f92df6b..0b32c4bb 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java @@ -68,12 +68,12 @@ private ConnectorsDriverSupport(String prefix, GenericSDMXClientSupplier supplie this.clock = clock; } - public SdmxConnection connect(URI uri, Map info, LanguagePriorityList languages) throws IOException { - return new ConnectorsConnection(getResource(uri, info, languages)); + public SdmxConnection connect(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + return new ConnectorsConnection(getResource(entryPoint, languages)); } - public boolean acceptsURI(URI uri) throws IOException { - return uri.toString().startsWith(prefix); + public boolean accepts(SdmxWebEntryPoint entryPoint) throws IOException { + return entryPoint.getUri().toString().startsWith(prefix); } @Override @@ -91,9 +91,10 @@ public static Collection entry(@Nonnull String name, @Nonnull return Collections.singleton(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); } - private ConnectorsConnection.Resource getResource(URI uri, Map info, LanguagePriorityList languages) throws IOException { + private ConnectorsConnection.Resource getResource(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + Map info = entryPoint.getProperties(); try { - URI endpoint = new URI(uri.toString().substring(prefix.length())); + URI endpoint = new URI(entryPoint.getUri().toString().substring(prefix.length())); GenericSDMXClient client = supplier.getClient(endpoint, info, languages); applyTimeouts(client, info); return CachedResource.of(client, endpoint, languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java index 00726132..1e7b5616 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java @@ -22,11 +22,9 @@ import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import java.io.IOException; -import java.net.URI; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.junit.Test; @@ -67,8 +65,8 @@ private static final class RepoDriver implements SdmxWebDriver { final List repos = Collections.singletonList(SdmxRepository.builder().name("r1").build()); @Override - public SdmxConnection connect(URI uri, Map properties, LanguagePriorityList languages) throws IOException { - String repoName = uri.toString().substring(PREFIX.length()); + public SdmxConnection connect(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + String repoName = entryPoint.getUri().toString().substring(PREFIX.length()); return repos.stream() .filter(o -> o.getName().equals(repoName)) .findFirst() @@ -77,8 +75,8 @@ public SdmxConnection connect(URI uri, Map properties, LanguagePriorityLis } @Override - public boolean acceptsURI(URI uri) throws IOException { - return uri.toString().startsWith(PREFIX); + public boolean accepts(SdmxWebEntryPoint entryPoint) throws IOException { + return entryPoint.getUri().toString().startsWith(PREFIX); } @Override From d639ffa21ff891a466278c4b9a7653b985729163 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 11:03:25 +0100 Subject: [PATCH 29/68] Improved SdmxFileProvider datasource label. --- .../java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java | 7 ++++--- .../main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java | 2 +- .../src/main/java/internal/sdmx/SdmxCubeAccessor.java | 7 ++++--- .../src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java | 6 ------ .../src/main/java/internal/file/FileSdmxConnection.java | 2 +- .../src/main/java/internal/file/SdmxFileUtil.java | 6 ++++++ .../test/java/internal/file/FileSdmxConnectionTest.java | 6 +++--- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index ff22dff7..50fa5c4a 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -39,6 +39,7 @@ import ec.tss.tsproviders.utils.DataSourcePreconditions; import ec.tss.tsproviders.utils.IParam; import ec.tstoolkit.utilities.GuavaCaches; +import internal.file.SdmxFileUtil; import internal.sdmx.SdmxCubeItems; import internal.sdmx.SdmxPropertiesSupport; import java.io.File; @@ -53,7 +54,7 @@ * @author Philippe Charles * @since 2.2.0 */ -//@ServiceProvider(service = ITsProvider.class) +//@org.openide.util.lookup.ServiceProvider(service = ITsProvider.class) public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private static final String NAME = "sdmx-file"; @@ -131,14 +132,14 @@ private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorit SdmxFile file = resolveFile(paths, bean.getFile(), bean.getStructureFile()); String source = file.toString(); - DataflowRef flow = file.getDataflowRef(); + DataflowRef flow = SdmxFileUtil.asDataflowRef(file); List dimensions = bean.getDimensions(); if (dimensions.isEmpty()) { dimensions = SdmxCubeItems.getDefaultDimIds(supplier, languages, source, flow); } - CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, source, flow, dimensions, bean.getLabelAttribute()); + CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, source, flow, dimensions, bean.getLabelAttribute(), bean.getFile().getPath()); IParam idParam = param.getCubeIdParam(accessor.getRoot()); diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index 53e6ec11..102da393 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -139,7 +139,7 @@ private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorit dimensions = SdmxCubeItems.getDefaultDimIds(supplier, languages, bean.getSource(), flow); } - CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, bean.getSource(), flow, dimensions, bean.getLabelAttribute()) + CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, bean.getSource(), flow, dimensions, bean.getLabelAttribute(), bean.getSource()) .bulk(bean.getCacheDepth(), GuavaCaches.ttlCacheAsMap(bean.getCacheTtl())); IParam idParam = param.getCubeIdParam(accessor.getRoot()); diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index 4d417a02..6ab3bac6 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -45,8 +45,8 @@ @lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) public final class SdmxCubeAccessor implements CubeAccessor { - public static SdmxCubeAccessor of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, String source, DataflowRef flow, List dimensions, String labelAttribute) { - return new SdmxCubeAccessor(supplier, languages, source, flow, CubeId.root(dimensions), labelAttribute); + public static SdmxCubeAccessor of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, String source, DataflowRef flow, List dimensions, String labelAttribute, String sourceLabel) { + return new SdmxCubeAccessor(supplier, languages, source, flow, CubeId.root(dimensions), labelAttribute, sourceLabel); } private final SdmxConnectionSupplier supplier; @@ -55,6 +55,7 @@ public static SdmxCubeAccessor of(SdmxConnectionSupplier supplier, LanguagePrior private final DataflowRef flowRef; private final CubeId root; private final String labelAttribute; + private final String sourceLabel; @Override public IOException testConnection() { @@ -117,7 +118,7 @@ public IteratorWithIO getChildren(CubeId ref) throws IOException { @Override public String getDisplayName() throws IOException { try (SdmxConnection conn = supplier.getConnection(source, languages)) { - return String.format("%s ~ %s", source, conn.getFlow(flowRef).getLabel()); + return String.format("%s ~ %s", sourceLabel, conn.getFlow(flowRef).getLabel()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java index 6063711a..d63bed43 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java @@ -16,7 +16,6 @@ */ package be.nbb.sdmx.facade.file; -import be.nbb.sdmx.facade.DataflowRef; import internal.file.SdmxFileUtil; import java.io.File; import javax.annotation.Nonnull; @@ -35,11 +34,6 @@ public class SdmxFile { @Nullable File structure; - @Nonnull - public DataflowRef getDataflowRef() { - return DataflowRef.parse("file"); - } - @Override public String toString() { return SdmxFileUtil.toXml(this); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index a73a3496..9e73dc35 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -59,7 +59,7 @@ class FileSdmxConnection implements SdmxConnection { this.languages = languages; this.factory = factory; this.decoder = decoder; - this.dataflow = Dataflow.of(file.getDataflowRef(), EMPTY, file.getData().getName()); + this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(file), EMPTY, file.getData().getName().replace(".xml", "")); this.closed = false; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index a3bec60e..42557170 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -16,6 +16,7 @@ */ package internal.file; +import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.file.SdmxFile; import java.io.File; import java.io.StringReader; @@ -34,6 +35,11 @@ @lombok.experimental.UtilityClass public class SdmxFileUtil { + @Nonnull + public DataflowRef asDataflowRef(@Nonnull SdmxFile file) { + return DataflowRef.parse("data" + (file.getStructure() != null ? "&struct" : "")); + } + @Nonnull public String toXml(@Nonnull SdmxFile file) { StringWriter result = new StringWriter(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 62439e0e..f1f89962 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -51,11 +51,11 @@ public void testCompactData21() throws IOException { FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); assertThat(conn.getFlows()).hasSize(1); - assertThat(conn.getStructure(file.getDataflowRef()).getDimensions()).hasSize(7); + assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(file)).getDimensions()).hasSize(7); Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getCursor(file.getDataflowRef(), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getCursor(SdmxFileUtil.asDataflowRef(file), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -76,6 +76,6 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), file.getDataflowRef()); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), SdmxFileUtil.asDataflowRef(file)); } } From 9e2230d0984f80a8d0dcd955948b7f6121335da6 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 11:44:08 +0100 Subject: [PATCH 30/68] Refactored SdmxFile as SdmxFileSet. --- .../demetra/sdmx/file/SdmxFileProvider.java | 12 +++++----- .../nbb/sdmx/facade/file/SdmxFileManager.java | 10 ++++---- .../file/{SdmxFile.java => SdmxFileSet.java} | 4 ++-- .../file/CachedFileSdmxConnection.java | 8 +++---- .../internal/file/FileSdmxConnection.java | 14 +++++------ .../main/java/internal/file/SdmxDecoder.java | 4 ++-- .../main/java/internal/file/SdmxFileUtil.java | 16 ++++++------- .../internal/file/XMLStreamSdmxDecoder.java | 12 +++++----- .../sdmx/facade/file/SdmxFileManagerTest.java | 4 ++-- ...SdmxFileTest.java => SdmxFileSetTest.java} | 24 +++++++++---------- .../internal/file/FileSdmxConnectionTest.java | 12 +++++----- 11 files changed, 60 insertions(+), 60 deletions(-) rename sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/{SdmxFile.java => SdmxFileSet.java} (90%) rename sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/{SdmxFileTest.java => SdmxFileSetTest.java} (70%) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 50fa5c4a..f2f1130d 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -22,7 +22,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.file.SdmxFileManager; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import com.google.common.cache.Cache; import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; @@ -129,10 +129,10 @@ private SdmxCubeItems get(DataSource dataSource) throws IOException { private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, HasFilePaths paths, SdmxFileParam param, DataSource dataSource) throws IOException { SdmxFileBean bean = param.get(dataSource); - SdmxFile file = resolveFile(paths, bean.getFile(), bean.getStructureFile()); + SdmxFileSet files = resolveFiles(paths, bean.getFile(), bean.getStructureFile()); - String source = file.toString(); - DataflowRef flow = SdmxFileUtil.asDataflowRef(file); + String source = files.toString(); + DataflowRef flow = SdmxFileUtil.asDataflowRef(files); List dimensions = bean.getDimensions(); if (dimensions.isEmpty()) { @@ -146,8 +146,8 @@ private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorit return new SdmxCubeItems(accessor, idParam); } - private static SdmxFile resolveFile(HasFilePaths paths, File data, File structure) throws FileNotFoundException { - return SdmxFile.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); + private static SdmxFileSet resolveFiles(HasFilePaths paths, File data, File structure) throws FileNotFoundException { + return SdmxFileSet.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); } } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 5a3ec94a..5582b647 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -50,20 +50,20 @@ public static SdmxFileManager of() { @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { - SdmxFile file; + SdmxFileSet files; try { - file = SdmxFile.parse(name); + files = SdmxFileSet.parse(name); } catch (IllegalArgumentException ex) { throw new IOException(ex.getMessage(), ex.getCause()); } - return getConnection(file, languages); + return getConnection(files, languages); } @Nonnull - public SdmxConnection getConnection(@Nonnull SdmxFile file, @Nonnull LanguagePriorityList languages) throws IOException { - return new CachedFileSdmxConnection(file, languages, factory, decoder, cache.get()); + public SdmxConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { + return new CachedFileSdmxConnection(files, languages, factory, decoder, cache.get()); } @Override diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java similarity index 90% rename from sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java rename to sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java index d63bed43..f4cebfb3 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFile.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java @@ -26,7 +26,7 @@ * @author Philippe Charles */ @lombok.Value(staticConstructor = "of") -public class SdmxFile { +public class SdmxFileSet { @lombok.NonNull File data; @@ -40,7 +40,7 @@ public String toString() { } @Nonnull - public static SdmxFile parse(@Nonnull String input) throws IllegalArgumentException { + public static SdmxFileSet parse(@Nonnull String input) throws IllegalArgumentException { return SdmxFileUtil.fromXml(input); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index b04cd2cd..3f752cec 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.util.TtlCache; @@ -46,10 +46,10 @@ public final class CachedFileSdmxConnection extends FileSdmxConnection { private final TypedId decodeKey; private final TypedId> loadDataKey; - public CachedFileSdmxConnection(SdmxFile file, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { - super(file, languages, factory, decoder); + public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { + super(files, languages, factory, decoder); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); - String base = file.toString() + languages.toString(); + String base = files.toString() + languages.toString(); this.decodeKey = TypedId.of("decode://" + base); this.loadDataKey = TypedId.of("loadData://" + base); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 9e73dc35..8e9dd5e3 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -27,7 +27,7 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; @@ -47,19 +47,19 @@ class FileSdmxConnection implements SdmxConnection { private static final DataStructureRef EMPTY = DataStructureRef.of("", "", ""); - private final SdmxFile file; + private final SdmxFileSet files; private final LanguagePriorityList languages; private final XMLInputFactory factory; private final SdmxDecoder decoder; private final Dataflow dataflow; private boolean closed; - FileSdmxConnection(SdmxFile file, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder) { - this.file = file; + FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder) { + this.files = files; this.languages = languages; this.factory = factory; this.decoder = decoder; - this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(file), EMPTY, file.getData().getName().replace(".xml", "")); + this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(files), EMPTY, files.getData().getName().replace(".xml", "")); this.closed = false; } @@ -116,11 +116,11 @@ private void checkState() throws IOException { } protected SdmxDecoder.Info decode() throws IOException { - return decoder.decode(file, languages); + return decoder.decode(files, languages); } protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return getDataSupplier(entry.getDataType(), entry.getDataStructure()).get(factory, file.getData().toPath(), StandardCharsets.UTF_8); + return getDataSupplier(entry.getDataType(), entry.getDataStructure()).get(factory, files.getData().toPath(), StandardCharsets.UTF_8); } private XMLStream getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java index c7f99794..f64008fa 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import java.io.IOException; import javax.annotation.Nonnull; @@ -29,7 +29,7 @@ public interface SdmxDecoder { @Nonnull - Info decode(@Nonnull SdmxFile file, @Nonnull LanguagePriorityList ranges) throws IOException; + Info decode(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList ranges) throws IOException; enum DataType { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 42557170..b9adf583 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -17,7 +17,7 @@ package internal.file; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import java.io.File; import java.io.StringReader; import java.io.StringWriter; @@ -36,18 +36,18 @@ public class SdmxFileUtil { @Nonnull - public DataflowRef asDataflowRef(@Nonnull SdmxFile file) { - return DataflowRef.parse("data" + (file.getStructure() != null ? "&struct" : "")); + public DataflowRef asDataflowRef(@Nonnull SdmxFileSet files) { + return DataflowRef.parse("data" + (files.getStructure() != null ? "&struct" : "")); } @Nonnull - public String toXml(@Nonnull SdmxFile file) { + public String toXml(@Nonnull SdmxFileSet files) { StringWriter result = new StringWriter(); try { XMLStreamWriter xml = OUTPUT.createXMLStreamWriter(result); xml.writeEmptyElement(ROOT_TAG); - xml.writeAttribute(DATA_ATTR, file.getData().toString()); - File structure = file.getStructure(); + xml.writeAttribute(DATA_ATTR, files.getData().toString()); + File structure = files.getStructure(); if (structure != null) { xml.writeAttribute(STRUCT_ATTR, structure.toString()); } @@ -60,7 +60,7 @@ public String toXml(@Nonnull SdmxFile file) { } @Nonnull - public static SdmxFile fromXml(@Nonnull String input) throws IllegalArgumentException { + public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentException { String data = null; String structure = null; try { @@ -78,7 +78,7 @@ public static SdmxFile fromXml(@Nonnull String input) throws IllegalArgumentExce if (data == null) { throw new IllegalArgumentException("Cannot parse SdmxFile from '" + input + "'"); } - return SdmxFile.of(new File(data), structure != null ? new File(structure) : null); + return SdmxFileSet.of(new File(data), structure != null ? new File(structure) : null); } private static final String ROOT_TAG = "file"; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index a4ded188..3b42644d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -26,7 +26,7 @@ import static internal.file.SdmxDecoder.DataType.COMPACT21; import static internal.file.SdmxDecoder.DataType.GENERIC20; import static internal.file.SdmxDecoder.DataType.GENERIC21; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.io.Reader; @@ -46,11 +46,11 @@ public XMLStreamSdmxDecoder(XMLInputFactory factory) { } @Override - public Info decode(SdmxFile file, LanguagePriorityList langs) throws IOException { - DataType dataType = probeDataType(file.getData()); - return Info.of(dataType, file.getStructure() != null - ? parseStruct(dataType, file.getStructure(), langs) - : decodeStruct(dataType, file.getData())); + public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { + DataType dataType = probeDataType(files.getData()); + return Info.of(dataType, files.getStructure() != null + ? parseStruct(dataType, files.getStructure(), langs) + : decodeStruct(dataType, files.getData())); } private DataType probeDataType(File data) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index 01ad51b4..70183c86 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -38,8 +38,8 @@ public void testCompliance() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFile file = SdmxFile.of(compact21, null); + SdmxFileSet files = SdmxFileSet.of(compact21, null); - ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), file.toString(), "ko"); + ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), files.toString(), "ko"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java similarity index 70% rename from sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java index e84f68b6..e64699b3 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java @@ -25,47 +25,47 @@ * * @author Philippe Charles */ -public class SdmxFileTest { +public class SdmxFileSetTest { @Test @SuppressWarnings("null") public void testFactory() { - assertThatThrownBy(() -> SdmxFile.of(null, null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxFileSet.of(null, null)).isInstanceOf(NullPointerException.class); - assertThat(SdmxFile.of(data, null)) + assertThat(SdmxFileSet.of(data, null)) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", null); - assertThat(SdmxFile.of(data, structure)) + assertThat(SdmxFileSet.of(data, structure)) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", structure); } @Test public void testToString() { - assertThat(SdmxFile.of(data, structure).toString()) + assertThat(SdmxFileSet.of(data, structure).toString()) .isEqualTo(""); - assertThat(SdmxFile.of(data, null).toString()) + assertThat(SdmxFileSet.of(data, null).toString()) .isEqualTo(""); } @Test @SuppressWarnings("null") public void testParse() { - assertThatThrownBy(() -> SdmxFile.parse(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SdmxFile.parse("")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SdmxFile.parse("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileSet.parse(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxFileSet.parse("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileSet.parse("")).isInstanceOf(IllegalArgumentException.class); - assertThat(SdmxFile.parse("")) + assertThat(SdmxFileSet.parse("")) .hasFieldOrPropertyWithValue("data", new File("")) .hasFieldOrPropertyWithValue("structure", null); - assertThat(SdmxFile.parse("")) + assertThat(SdmxFileSet.parse("")) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", null); - assertThat(SdmxFile.parse("")) + assertThat(SdmxFileSet.parse("")) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", structure); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index f1f89962..f88a2c51 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -21,7 +21,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.file.SdmxFile; +import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; import java.io.File; @@ -46,16 +46,16 @@ public void testCompactData21() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFile file = SdmxFile.of(compact21, null); + SdmxFileSet files = SdmxFileSet.of(compact21, null); - FileSdmxConnection conn = new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); + FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); assertThat(conn.getFlows()).hasSize(1); - assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(file)).getDimensions()).hasSize(7); + assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(files)).getDimensions()).hasSize(7); Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getCursor(SdmxFileUtil.asDataflowRef(file), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getCursor(SdmxFileUtil.asDataflowRef(files), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -76,6 +76,6 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(file, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), SdmxFileUtil.asDataflowRef(file)); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), SdmxFileUtil.asDataflowRef(files)); } } From 29babc78394b470c5e498ca5971d5aad7f3b367c Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 12:55:26 +0100 Subject: [PATCH 31/68] Improved use of connection factories. --- .../nbb/demetra/dotstat/DotStatProvider.java | 42 ++++--------------- .../demetra/sdmx/file/SdmxFileProvider.java | 31 +++++++++----- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 27 +++++++----- .../java/internal/sdmx/SdmxCubeAccessor.java | 28 +++++-------- .../java/internal/sdmx/SdmxCubeItems.java | 20 ++++++--- .../internal/sdmx/SdmxPropertiesSupport.java | 2 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 3 +- .../be/nbb/sdmx/facade/file/SdmxFileSet.java | 12 ------ .../file/CachedFileSdmxConnection.java | 2 +- .../sdmx/facade/file/SdmxFileManagerTest.java | 3 +- .../nbb/sdmx/facade/file/SdmxFileSetTest.java | 17 ++++---- 11 files changed, 86 insertions(+), 101 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java index c2587bf8..ecbfbbd3 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/dotstat/DotStatProvider.java @@ -22,7 +22,6 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.collect.Maps; import ec.tss.ITsProvider; @@ -32,9 +31,9 @@ import ec.tss.tsproviders.db.DbAccessor; import ec.tss.tsproviders.db.DbBean; import ec.tss.tsproviders.db.DbProvider; +import internal.sdmx.SdmxPropertiesSupport; import java.io.IOException; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.openide.util.lookup.ServiceProvider; @@ -49,20 +48,21 @@ public final class DotStatProvider extends DbProvider implements HasSdmxProperties { public static final String NAME = "DOTSTAT", VERSION = "20150203"; - private final AtomicReference connectionSupplier; - private final AtomicReference languages; + + @lombok.experimental.Delegate + private final HasSdmxProperties properties; + private boolean displayCodes; public DotStatProvider() { super(LoggerFactory.getLogger(DotStatProvider.class), NAME, TsAsyncMode.Once); - this.connectionSupplier = new AtomicReference<>(SdmxWebManager.ofServiceLoader()); - this.languages = new AtomicReference<>(LanguagePriorityList.ANY); + this.properties = SdmxPropertiesSupport.of(SdmxWebManager::ofServiceLoader, this::clearCache, () -> LanguagePriorityList.ANY, this::clearCache); this.displayCodes = false; } @Override protected DbAccessor loadFromBean(DotStatBean bean) throws Exception { - return new DotStatAccessor(bean, connectionSupplier.get(), languages.get()).memoize(); + return new DotStatAccessor(bean, getConnectionSupplier(), getLanguages()).memoize(); } @Override @@ -137,32 +137,6 @@ public DataSource encodeBean(Object bean) throws IllegalArgumentException { return support.checkBean(bean, DotStatBean.class).toDataSource(NAME, VERSION); } - @Override - public SdmxConnectionSupplier getConnectionSupplier() { - return connectionSupplier.get(); - } - - @Override - public void setConnectionSupplier(SdmxConnectionSupplier connectionSupplier) { - SdmxConnectionSupplier old = this.connectionSupplier.get(); - if (this.connectionSupplier.compareAndSet(old, connectionSupplier != null ? connectionSupplier : SdmxWebManager.ofServiceLoader())) { - clearCache(); - } - } - - @Override - public LanguagePriorityList getLanguages() { - return languages.get(); - } - - @Override - public void setLanguages(LanguagePriorityList languages) { - LanguagePriorityList old = this.languages.get(); - if (this.languages.compareAndSet(old, languages != null ? languages : LanguagePriorityList.ANY)) { - clearCache(); - } - } - @Nonnull public String getPreferredLanguage() { return getLanguages().toString(); @@ -184,7 +158,7 @@ public void setDisplayCodes(boolean displayCodes) { } private SdmxConnection connect(String name) throws IOException { - return connectionSupplier.get().getConnection(name, languages.get()); + return getConnectionSupplier().getConnection(name, getLanguages()); } @Nullable diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index f2f1130d..b8fdeb71 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -20,9 +20,11 @@ import internal.sdmx.SdmxCubeAccessor; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.file.SdmxFileManager; import be.nbb.sdmx.facade.file.SdmxFileSet; +import be.nbb.sdmx.facade.util.IO; import com.google.common.cache.Cache; import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; @@ -45,7 +47,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +61,7 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { private static final String NAME = "sdmx-file"; @lombok.experimental.Delegate - private final SdmxPropertiesSupport properties; + private final HasSdmxProperties properties; @lombok.experimental.Delegate private final HasDataSourceMutableList mutableListSupport; @@ -124,28 +125,38 @@ public IParam getIdParam(DataSource dataSource) throws IOExcept private SdmxCubeItems get(DataSource dataSource) throws IOException { DataSourcePreconditions.checkProvider(NAME, dataSource); - return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties.getConnectionSupplier(), properties.getLanguages(), paths, param, dataSource)); + return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties, paths, param, dataSource)); } - private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, HasFilePaths paths, SdmxFileParam param, DataSource dataSource) throws IOException { + private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths, SdmxFileParam param, DataSource dataSource) throws IOException { SdmxFileBean bean = param.get(dataSource); SdmxFileSet files = resolveFiles(paths, bean.getFile(), bean.getStructureFile()); - String source = files.toString(); DataflowRef flow = SdmxFileUtil.asDataflowRef(files); - List dimensions = bean.getDimensions(); - if (dimensions.isEmpty()) { - dimensions = SdmxCubeItems.getDefaultDimIds(supplier, languages, source, flow); - } + IO.Supplier conn = getSupplier(properties, files); + + CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), conn, flow); - CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, source, flow, dimensions, bean.getLabelAttribute(), bean.getFile().getPath()); + CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), bean.getFile().getPath()); IParam idParam = param.getCubeIdParam(accessor.getRoot()); return new SdmxCubeItems(accessor, idParam); } + private static IO.Supplier getSupplier(HasSdmxProperties properties, SdmxFileSet files) { + SdmxConnectionSupplier supplier = properties.getConnectionSupplier(); + LanguagePriorityList languages = properties.getLanguages(); + + if (supplier instanceof SdmxFileManager) { + return () -> ((SdmxFileManager) supplier).getConnection(files, languages); + } + + String name = SdmxFileUtil.toXml(files); + return () -> supplier.getConnection(name, languages); + } + private static SdmxFileSet resolveFiles(HasFilePaths paths, File data, File structure) throws FileNotFoundException { return SdmxFileSet.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index 102da393..a6ce365a 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -20,7 +20,9 @@ import internal.sdmx.SdmxCubeAccessor; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.util.IO; import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.cache.Cache; import ec.tss.ITsProvider; @@ -40,7 +42,6 @@ import internal.sdmx.SdmxCubeItems; import internal.sdmx.SdmxPropertiesSupport; import java.io.IOException; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.openide.util.lookup.ServiceProvider; import org.slf4j.Logger; @@ -59,7 +60,7 @@ public final class SdmxWebProvider implements IDataSourceLoader, HasSdmxProperti private final AtomicBoolean displayCodes; @lombok.experimental.Delegate - private final SdmxPropertiesSupport properties; + private final HasSdmxProperties properties; @lombok.experimental.Delegate private final HasDataSourceMutableList mutableListSupport; @@ -111,7 +112,7 @@ public void setDisplayCodes(boolean displayCodes) { private static final class SdmxCubeResource implements CubeSupport.Resource { private final Cache cache; - private final SdmxPropertiesSupport properties; + private final HasSdmxProperties properties; private final SdmxWebParam param; @Override @@ -126,25 +127,31 @@ public IParam getIdParam(DataSource dataSource) throws IOExcept private SdmxCubeItems get(DataSource dataSource) throws IOException { DataSourcePreconditions.checkProvider(NAME, dataSource); - return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties.getConnectionSupplier(), properties.getLanguages(), param, dataSource)); + return GuavaCaches.getOrThrowIOException(cache, dataSource, () -> of(properties, param, dataSource)); } - private static SdmxCubeItems of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, SdmxWebParam param, DataSource dataSource) throws IllegalArgumentException, IOException { + private static SdmxCubeItems of(HasSdmxProperties properties, SdmxWebParam param, DataSource dataSource) throws IllegalArgumentException, IOException { SdmxWebBean bean = param.get(dataSource); DataflowRef flow = DataflowRef.parse(bean.getFlow()); - List dimensions = bean.getDimensions(); - if (dimensions.isEmpty()) { - dimensions = SdmxCubeItems.getDefaultDimIds(supplier, languages, bean.getSource(), flow); - } + IO.Supplier conn = getSupplier(properties, bean.getSource()); - CubeAccessor accessor = SdmxCubeAccessor.of(supplier, languages, bean.getSource(), flow, dimensions, bean.getLabelAttribute(), bean.getSource()) + CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), conn, flow); + + CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), bean.getSource()) .bulk(bean.getCacheDepth(), GuavaCaches.ttlCacheAsMap(bean.getCacheTtl())); IParam idParam = param.getCubeIdParam(accessor.getRoot()); return new SdmxCubeItems(accessor, idParam); } + + private static IO.Supplier getSupplier(HasSdmxProperties properties, String name) { + SdmxConnectionSupplier supplier = properties.getConnectionSupplier(); + LanguagePriorityList languages = properties.getLanguages(); + + return () -> supplier.getConnection(name, languages); + } } } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index 6ab3bac6..8c8a4d67 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -20,9 +20,8 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import com.google.common.base.Converter; import com.google.common.collect.Maps; @@ -36,22 +35,15 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; -import lombok.AccessLevel; /** * * @author Philippe Charles */ -@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +@lombok.AllArgsConstructor(staticName = "of") public final class SdmxCubeAccessor implements CubeAccessor { - public static SdmxCubeAccessor of(SdmxConnectionSupplier supplier, LanguagePriorityList languages, String source, DataflowRef flow, List dimensions, String labelAttribute, String sourceLabel) { - return new SdmxCubeAccessor(supplier, languages, source, flow, CubeId.root(dimensions), labelAttribute, sourceLabel); - } - - private final SdmxConnectionSupplier supplier; - private final LanguagePriorityList languages; - private final String source; + private final IO.Supplier supplier; private final DataflowRef flowRef; private final CubeId root; private final String labelAttribute; @@ -69,7 +61,7 @@ public CubeId getRoot() { @Override public TsCursor getAllSeries(CubeId ref) throws IOException { - SdmxConnection conn = supplier.getConnection(source, languages); + SdmxConnection conn = supplier.getWithIO(); try { return getAllSeriesCursor(conn, flowRef, ref, labelAttribute).onClose(conn); } catch (IOException ex) { @@ -81,7 +73,7 @@ public TsCursor getAllSeries(CubeId ref) throws IOException { @Override public TsCursor getAllSeriesWithData(CubeId ref) throws IOException { - SdmxConnection conn = supplier.getConnection(source, languages); + SdmxConnection conn = supplier.getWithIO(); try { return getAllSeriesWithDataCursor(conn, flowRef, ref, labelAttribute).onClose(conn); } catch (IOException ex) { @@ -93,7 +85,7 @@ public TsCursor getAllSeriesWithData(CubeId ref) throws IOException { @Override public TsCursor getSeriesWithData(CubeId ref) throws IOException { - SdmxConnection conn = supplier.getConnection(source, languages); + SdmxConnection conn = supplier.getWithIO(); try { return getSeriesWithDataCursor(conn, flowRef, ref, labelAttribute).onClose(conn); } catch (IOException ex) { @@ -105,7 +97,7 @@ public TsCursor getSeriesWithData(CubeId ref) throws IOException { @Override public IteratorWithIO getChildren(CubeId ref) throws IOException { - SdmxConnection conn = supplier.getConnection(source, languages); + SdmxConnection conn = supplier.getWithIO(); try { return getChildren(conn, flowRef, ref).onClose(conn); } catch (IOException ex) { @@ -117,7 +109,7 @@ public IteratorWithIO getChildren(CubeId ref) throws IOException { @Override public String getDisplayName() throws IOException { - try (SdmxConnection conn = supplier.getConnection(source, languages)) { + try (SdmxConnection conn = supplier.getWithIO()) { return String.format("%s ~ %s", sourceLabel, conn.getFlow(flowRef).getLabel()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); @@ -129,7 +121,7 @@ public String getDisplayName(CubeId id) throws IOException { if (id.isVoid()) { return "All"; } - try (SdmxConnection conn = supplier.getConnection(source, languages)) { + try (SdmxConnection conn = supplier.getWithIO()) { Map dimensionById = dimensionById(conn.getStructure(flowRef)); return getKey(dimensionById, id).toString(); } catch (RuntimeException ex) { @@ -142,7 +134,7 @@ public String getDisplayNodeName(CubeId id) throws IOException { if (id.isVoid()) { return "All"; } - try (SdmxConnection conn = supplier.getConnection(source, languages)) { + try (SdmxConnection conn = supplier.getWithIO()) { Map dimensionById = dimensionById(conn.getStructure(flowRef)); return getDisplayNodeName(dimensionById, id); } catch (RuntimeException ex) { diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index 3517e7d3..2399bd18 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -18,9 +18,8 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.cube.CubeAccessor; @@ -40,9 +39,20 @@ public class SdmxCubeItems { CubeAccessor accessor; IParam idParam; - public static List getDefaultDimIds(SdmxConnectionSupplier supplier, LanguagePriorityList languages, String source, DataflowRef flow) throws IOException { - try (SdmxConnection conn = supplier.getConnection(source, languages)) { - return conn.getStructure(flow).getDimensions().stream().map(Dimension::getId).collect(Collectors.toList()); + public static CubeId getOrLoadRoot(List dimensions, IO.Supplier supplier, DataflowRef flow) throws IOException { + return dimensions.isEmpty() + ? CubeId.root(loadDefaultDimIds(supplier, flow)) + : CubeId.root(dimensions); + } + + public static List loadDefaultDimIds(IO.Supplier supplier, DataflowRef flow) throws IOException { + try (SdmxConnection conn = supplier.getWithIO()) { + return conn + .getStructure(flow) + .getDimensions() + .stream() + .map(Dimension::getId) + .collect(Collectors.toList()); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java index 6ef070cd..26ec7f27 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxPropertiesSupport.java @@ -30,7 +30,7 @@ @lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) public final class SdmxPropertiesSupport implements HasSdmxProperties { - public static SdmxPropertiesSupport of(Supplier defaultSupplier, Runnable onSupplierChange, Supplier defaultLanguages, Runnable onLanguagesChange) { + public static HasSdmxProperties of(Supplier defaultSupplier, Runnable onSupplierChange, Supplier defaultLanguages, Runnable onLanguagesChange) { return new SdmxPropertiesSupport( defaultSupplier, new AtomicReference<>(defaultSupplier.get()), diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 5582b647..c7689df7 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; +import internal.file.SdmxFileUtil; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -53,7 +54,7 @@ public SdmxConnection getConnection(String name, LanguagePriorityList languages) SdmxFileSet files; try { - files = SdmxFileSet.parse(name); + files = SdmxFileUtil.fromXml(name); } catch (IllegalArgumentException ex) { throw new IOException(ex.getMessage(), ex.getCause()); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java index f4cebfb3..32695001 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java @@ -16,9 +16,7 @@ */ package be.nbb.sdmx.facade.file; -import internal.file.SdmxFileUtil; import java.io.File; -import javax.annotation.Nonnull; import javax.annotation.Nullable; /** @@ -33,14 +31,4 @@ public class SdmxFileSet { @Nullable File structure; - - @Override - public String toString() { - return SdmxFileUtil.toXml(this); - } - - @Nonnull - public static SdmxFileSet parse(@Nonnull String input) throws IllegalArgumentException { - return SdmxFileUtil.fromXml(input); - } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 3f752cec..f5a40846 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -49,7 +49,7 @@ public final class CachedFileSdmxConnection extends FileSdmxConnection { public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { super(files, languages, factory, decoder); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); - String base = files.toString() + languages.toString(); + String base = SdmxFileUtil.toXml(files) + languages.toString(); this.decodeKey = TypedId.of("decode://" + base); this.loadDataKey = TypedId.of("loadData://" + base); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index 70183c86..bf9b59e2 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -18,6 +18,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; +import internal.file.SdmxFileUtil; import java.io.File; import java.io.IOException; import org.junit.Rule; @@ -40,6 +41,6 @@ public void testCompliance() throws IOException { SdmxFileSet files = SdmxFileSet.of(compact21, null); - ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), files.toString(), "ko"); + ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), SdmxFileUtil.toXml(files), "ko"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java index e64699b3..ab835d8d 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.file; +import internal.file.SdmxFileUtil; import java.io.File; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -43,29 +44,29 @@ public void testFactory() { @Test public void testToString() { - assertThat(SdmxFileSet.of(data, structure).toString()) + assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, structure))) .isEqualTo(""); - assertThat(SdmxFileSet.of(data, null).toString()) + assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, null))) .isEqualTo(""); } @Test @SuppressWarnings("null") public void testParse() { - assertThatThrownBy(() -> SdmxFileSet.parse(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SdmxFileSet.parse("")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SdmxFileSet.parse("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); - assertThat(SdmxFileSet.parse("")) + assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", new File("")) .hasFieldOrPropertyWithValue("structure", null); - assertThat(SdmxFileSet.parse("")) + assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", null); - assertThat(SdmxFileSet.parse("")) + assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) .hasFieldOrPropertyWithValue("structure", structure); } From 162e798e0480c048cf28bafb6dabe8ef3edacc02 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 14:41:36 +0100 Subject: [PATCH 32/68] Fixed cache initialization in SdmxWebProviderBuddy. --- .../main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java | 1 + 1 file changed, 1 insertion(+) diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java index a5dc5cfa..ee8e0657 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java @@ -79,6 +79,7 @@ private static SdmxWebManager createManager() { public SdmxWebProviderBuddy() { this.configurator = createConfigurator(); this.autoCompletionCache = GuavaCaches.ttlCacheAsMap(Duration.ofMinutes(1)); + lookupProvider().ifPresent(o -> o.setConnectionSupplier(WEB_MANAGER)); } @Override From 0eb8feef6ea8e1dfcdf6228fa8d82b32805b1608 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 9 Nov 2017 16:43:33 +0100 Subject: [PATCH 33/68] Added SdmxFileProvider as a registered provider. --- .../demetra/sdmx/file/SdmxFileProvider.java | 17 +- .../java/internal/sdmx/SdmxCubeItems.java | 22 ++ demetra-dotstat-desktop/pom.xml | 4 + .../sdmx/file/SdmxFileProviderBuddy.java | 243 ++++++++++++++++++ .../internal/sdmx/LegacySdmxProvider.java | 34 +++ .../main/java/internal/file/SdmxFileUtil.java | 7 +- .../nbb/sdmx/facade/file/SdmxFileSetTest.java | 30 --- .../java/internal/file/SdmxFileUtilTest.java | 75 ++++++ 8 files changed, 391 insertions(+), 41 deletions(-) create mode 100644 demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java create mode 100644 demetra-dotstat-desktop/src/main/java/internal/sdmx/LegacySdmxProvider.java create mode 100644 sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index b8fdeb71..6b294087 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -45,8 +45,8 @@ import internal.sdmx.SdmxCubeItems; import internal.sdmx.SdmxPropertiesSupport; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; +import org.openide.util.lookup.ServiceProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,10 +55,10 @@ * @author Philippe Charles * @since 2.2.0 */ -//@org.openide.util.lookup.ServiceProvider(service = ITsProvider.class) +@ServiceProvider(service = ITsProvider.class) public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { - private static final String NAME = "sdmx-file"; + public static final String NAME = "sdmx-file"; @lombok.experimental.Delegate private final HasSdmxProperties properties; @@ -95,6 +95,11 @@ public SdmxFileProvider() { this.tsSupport = CubeSupport.asTsProvider(NAME, logger, cubeSupport, monikerSupport, cache::invalidateAll); } + @Override + public String getDisplayName() { + return "SDMX Files"; + } + @Override public String getFileDescription() { return "SDMX file"; @@ -130,7 +135,7 @@ private SdmxCubeItems get(DataSource dataSource) throws IOException { private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths, SdmxFileParam param, DataSource dataSource) throws IOException { SdmxFileBean bean = param.get(dataSource); - SdmxFileSet files = resolveFiles(paths, bean.getFile(), bean.getStructureFile()); + SdmxFileSet files = SdmxCubeItems.resolveFileSet(paths, bean); DataflowRef flow = SdmxFileUtil.asDataflowRef(files); @@ -156,9 +161,5 @@ private static IO.Supplier getSupplier(HasSdmxProperties propert String name = SdmxFileUtil.toXml(files); return () -> supplier.getConnection(name, languages); } - - private static SdmxFileSet resolveFiles(HasFilePaths paths, File data, File structure) throws FileNotFoundException { - return SdmxFileSet.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); - } } } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index 2399bd18..924891ad 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -16,17 +16,23 @@ */ package internal.sdmx; +import be.nbb.demetra.sdmx.file.SdmxFileBean; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import ec.tss.tsproviders.DataSet; +import ec.tss.tsproviders.HasFilePaths; import ec.tss.tsproviders.cube.CubeAccessor; import ec.tss.tsproviders.cube.CubeId; import ec.tss.tsproviders.utils.IParam; +import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -57,4 +63,20 @@ public static List loadDefaultDimIds(IO.Supplier supplie throw new UnexpectedIOException(ex); } } + + public static Optional tryResolveFileSet(HasFilePaths paths, SdmxFileBean bean) { + try { + return Optional.of(resolveFileSet(paths, bean)); + } catch (FileNotFoundException ex) { + return Optional.empty(); + } + } + + public static SdmxFileSet resolveFileSet(HasFilePaths paths, SdmxFileBean bean) throws FileNotFoundException { + return resolveFileSet(paths, bean.getFile(), bean.getStructureFile()); + } + + public static SdmxFileSet resolveFileSet(HasFilePaths paths, File data, File structure) throws FileNotFoundException { + return SdmxFileSet.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); + } } diff --git a/demetra-dotstat-desktop/pom.xml b/demetra-dotstat-desktop/pom.xml index 6a9d6f7d..19f962ea 100644 --- a/demetra-dotstat-desktop/pom.xml +++ b/demetra-dotstat-desktop/pom.xml @@ -113,6 +113,10 @@ eu.europa.ec.joinup.sat nbdemetra-jdbc + + eu.europa.ec.joinup.sat + nbdemetra-sdmx + be.nbb.demetra diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java new file mode 100644 index 00000000..2bf1e274 --- /dev/null +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java @@ -0,0 +1,243 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.demetra.sdmx.file; + +import be.nbb.demetra.dotstat.DotStatOptionsPanelController; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.file.SdmxFileManager; +import com.google.common.base.Converter; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import ec.nbdemetra.ui.BeanHandler; +import ec.nbdemetra.ui.Config; +import ec.nbdemetra.ui.Configurator; +import ec.nbdemetra.ui.IConfigurable; +import ec.nbdemetra.ui.properties.FileLoaderFileFilter; +import ec.nbdemetra.ui.properties.NodePropertySetBuilder; +import ec.nbdemetra.ui.properties.PropertySheetDialogBuilder; +import ec.nbdemetra.ui.tsproviders.IDataSourceProviderBuddy; +import ec.tss.tsproviders.DataSet; +import ec.tss.tsproviders.IFileLoader; +import ec.tss.tsproviders.TsProviders; +import ec.tstoolkit.utilities.GuavaCaches; +import internal.file.SdmxFileUtil; +import internal.sdmx.SdmxAutoCompletion; +import internal.sdmx.SdmxCubeItems; +import java.awt.Image; +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; +import org.netbeans.api.options.OptionsDisplayer; +import org.openide.nodes.Sheet; +import org.openide.util.ImageUtilities; +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author Philippe Charles + * @since 2.2.1 + */ +@ServiceProvider(service = IDataSourceProviderBuddy.class) +public final class SdmxFileProviderBuddy implements IDataSourceProviderBuddy, IConfigurable { + + private final Configurator configurator; + private final ConcurrentMap autoCompletionCache; + + public SdmxFileProviderBuddy() { + this.configurator = createConfigurator(); + this.autoCompletionCache = GuavaCaches.ttlCacheAsMap(Duration.ofMinutes(1)); + lookupProvider().ifPresent(o -> o.setConnectionSupplier(createManager())); + } + + @Override + public String getProviderName() { + return SdmxFileProvider.NAME; + } + + @Override + public Image getIcon(int type, boolean opened) { + return ImageUtilities.loadImage("ec/nbdemetra/sdmx/document-code.png", true); + } + + @Override + public Image getIcon(DataSet dataSet, int type, boolean opened) { + switch (dataSet.getKind()) { + case COLLECTION: + return ImageUtilities.loadImage("ec/nbdemetra/ui/nodes/folder.png", true); + case SERIES: + return ImageUtilities.loadImage("ec/nbdemetra/ui/nodes/chart_line.png", true); + case DUMMY: + return null; + } + return IDataSourceProviderBuddy.super.getIcon(dataSet, type, opened); + } + + @Override + public Image getIcon(IOException ex, int type, boolean opened) { + return ImageUtilities.loadImage("ec/nbdemetra/ui/nodes/exclamation-red.png", true); + } + + @Override + public boolean editBean(String title, Object bean) throws IntrospectionException { + if (bean instanceof SdmxFileBean) { + Optional provider = lookupProvider(); + if (provider.isPresent()) { + return editBean(title, (SdmxFileBean) bean, provider.get()); + } + } + return IDataSourceProviderBuddy.super.editBean(title, bean); + } + + @Override + public Config getConfig() { + return configurator.getConfig(this); + } + + @Override + public void setConfig(Config config) throws IllegalArgumentException { + configurator.setConfig(this, config); + } + + @Override + public Config editConfig(Config config) throws IllegalArgumentException { + OptionsDisplayer.getDefault().open(DotStatOptionsPanelController.ID); + return config; + } + + // + private static SdmxFileManager createManager() { + SdmxFileManager result = SdmxFileManager.of(); + result.setCache(GuavaCaches.softValuesCacheAsMap()); + return result; + } + + private static Optional lookupProvider() { + return TsProviders.lookup(SdmxFileProvider.class, SdmxFileProvider.NAME).toJavaUtil(); + } + + private static Configurator createConfigurator() { + return new BuddyConfigHandler().toConfigurator(new BuddyConfigConverter()); + } + + private static final class BuddyConfig { + + } + + private static final class BuddyConfigHandler extends BeanHandler { + + @Override + public BuddyConfig loadBean(SdmxFileProviderBuddy resource) { + return new BuddyConfig(); + } + + @Override + public void storeBean(SdmxFileProviderBuddy resource, BuddyConfig bean) { + } + } + + private static final class BuddyConfigConverter extends Converter { + + @Override + protected Config doForward(BuddyConfig a) { + return Config.builder(Void.class.getName(), "INSTANCE", "2017").build(); + } + + @Override + protected BuddyConfig doBackward(Config b) { + return new BuddyConfig(); + } + } + + private boolean editBean(String title, SdmxFileBean bean, SdmxFileProvider o) { + return new PropertySheetDialogBuilder() + .title(title) + .icon(getIcon(BeanInfo.ICON_COLOR_16x16, false)) + .editSheet(createSheet(bean, o, o.getConnectionSupplier(), o.getLanguages())); + } + + @NbBundle.Messages({ + "bean.cache.description=Mechanism used to improve performance."}) + private Sheet createSheet(SdmxFileBean bean, IFileLoader loader, SdmxConnectionSupplier supplier, LanguagePriorityList languages) { + Sheet result = new Sheet(); + NodePropertySetBuilder b = new NodePropertySetBuilder(); + result.put(withSource(b.reset("Source"), bean, loader).build()); + result.put(withOptions(b.reset("Options"), bean, loader, supplier, languages).build()); + return result; + } + + @NbBundle.Messages({ + "bean.file.display=Sdmx data file", + "bean.file.description=The path to the sdmx data file.",}) + private NodePropertySetBuilder withSource(NodePropertySetBuilder b, SdmxFileBean bean, IFileLoader loader) { + b.withFile() + .select(bean, "file") + .display(Bundle.bean_file_display()) + .description(Bundle.bean_file_description()) + .filterForSwing(new FileLoaderFileFilter(loader)) + .paths(loader.getPaths()) + .directories(false) + .add(); + return b; + } + + @NbBundle.Messages({ + "bean.structureFile.display=Sdmx structure file", + "bean.structureFile.description=The path to the sdmx structure file.", + "bean.dimensions.display=Dataflow dimensions", + "bean.dimensions.description=An optional comma-separated list of dimensions that defines the order used to hierarchise time series.", + "bean.labelAttribute.display=Series label attribute", + "bean.labelAttribute.description=An optional attribute that carries the label of time series." + }) + private NodePropertySetBuilder withOptions(NodePropertySetBuilder b, SdmxFileBean bean, IFileLoader loader, SdmxConnectionSupplier supplier, LanguagePriorityList languages) { + b.withFile() + .select(bean, "structureFile") + .display(Bundle.bean_structureFile_display()) + .description(Bundle.bean_structureFile_description()) + .filterForSwing(new FileLoaderFileFilter(loader)) + .paths(loader.getPaths()) + .directories(false) + .add(); + + Supplier toSource = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileUtil::toXml).orElse(""); + Supplier toFlow = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileUtil::asDataflowRef).map(Object::toString).orElse(""); + + b.withAutoCompletion() + .select(bean, "dimensions", List.class, Joiner.on(',')::join, Splitter.on(',').trimResults().omitEmptyStrings()::splitToList) + .source(SdmxAutoCompletion.onDimensions(supplier, languages, toSource, toFlow, autoCompletionCache)) + .separator(",") + .defaultValueSupplier(() -> SdmxAutoCompletion.getDefaultDimensionsAsString(supplier, languages, toSource, toFlow, autoCompletionCache, ",")) + .cellRenderer(SdmxAutoCompletion.getDimensionsRenderer()) + .display(Bundle.bean_dimensions_display()) + .description(Bundle.bean_dimensions_description()) + .add(); + + b.withAutoCompletion() + .select(bean, "labelAttribute") + .display(Bundle.bean_labelAttribute_display()) + .description(Bundle.bean_labelAttribute_description()) + .add(); + return b; + } + // +} diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/LegacySdmxProvider.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/LegacySdmxProvider.java new file mode 100644 index 00000000..948f3f08 --- /dev/null +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/LegacySdmxProvider.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.sdmx; + +import ec.tss.ITsProvider; +import ec.tss.tsproviders.sdmx.SdmxProvider; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author Philippe Charles + */ +@ServiceProvider(service = ITsProvider.class, supersedes = "ec.tss.tsproviders.sdmx.SdmxProvider") +public final class LegacySdmxProvider extends SdmxProvider { + + @Override + public String getDisplayName() { + return "SDMX Files (deprecated)"; + } +} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index b9adf583..23cc560d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -37,7 +37,8 @@ public class SdmxFileUtil { @Nonnull public DataflowRef asDataflowRef(@Nonnull SdmxFileSet files) { - return DataflowRef.parse("data" + (files.getStructure() != null ? "&struct" : "")); + File structFile = files.getStructure(); + return DataflowRef.parse("data" + (structFile != null && !structFile.toString().isEmpty() ? "&struct" : "")); } @Nonnull @@ -75,10 +76,10 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE } catch (XMLStreamException ex) { throw new IllegalArgumentException("Cannot parse SdmxFile", ex); } - if (data == null) { + if (data == null || data.isEmpty()) { throw new IllegalArgumentException("Cannot parse SdmxFile from '" + input + "'"); } - return SdmxFileSet.of(new File(data), structure != null ? new File(structure) : null); + return SdmxFileSet.of(new File(data), structure != null && !structure.isEmpty() ? new File(structure) : null); } private static final String ROOT_TAG = "file"; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java index ab835d8d..2f93ed1b 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java @@ -16,7 +16,6 @@ */ package be.nbb.sdmx.facade.file; -import internal.file.SdmxFileUtil; import java.io.File; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -42,35 +41,6 @@ public void testFactory() { .hasFieldOrPropertyWithValue("structure", structure); } - @Test - public void testToString() { - assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, structure))) - .isEqualTo(""); - - assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, null))) - .isEqualTo(""); - } - - @Test - @SuppressWarnings("null") - public void testParse() { - assertThatThrownBy(() -> SdmxFileUtil.fromXml(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); - - assertThat(SdmxFileUtil.fromXml("")) - .hasFieldOrPropertyWithValue("data", new File("")) - .hasFieldOrPropertyWithValue("structure", null); - - assertThat(SdmxFileUtil.fromXml("")) - .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", null); - - assertThat(SdmxFileUtil.fromXml("")) - .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", structure); - } - private final File data = new File("a.xml"); private final File structure = new File("b.xml"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java new file mode 100644 index 00000000..c172d16e --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.file; + +import be.nbb.sdmx.facade.file.SdmxFileSet; +import java.io.File; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxFileUtilTest { + + @Test + public void testAsDataflowRef() { + assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, structure)).toString()) + .isEqualTo("all,data&struct,latest"); + + assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, new File(""))).toString()) + .isEqualTo("all,data,latest"); + + assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, null)).toString()) + .isEqualTo("all,data,latest"); + } + + @Test + public void testToXml() { + assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, structure))) + .isEqualTo(""); + + assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, null))) + .isEqualTo(""); + } + + @Test + @SuppressWarnings("null") + public void testFromXml() { + assertThatThrownBy(() -> SdmxFileUtil.fromXml(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); + + assertThat(SdmxFileUtil.fromXml("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", null); + + assertThat(SdmxFileUtil.fromXml("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", null); + + assertThat(SdmxFileUtil.fromXml("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", structure); + } + + private final File data = new File("a.xml"); + private final File structure = new File("b.xml"); +} From 0e994e68a2d7849ad20779dd6a7789633c69810d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 14 Nov 2017 14:00:59 +0100 Subject: [PATCH 34/68] Code cleanup. --- .../internal/file/DataStructureDecoder.java | 60 ++++++++++------- .../java/internal/file/DataTypeProbe.java | 39 +++++------ .../internal/file/XMLStreamSdmxDecoder.java | 10 +-- .../file/DataStructureDecoderTest.java | 8 +-- .../java/internal/file/DataTypeProbeTest.java | 8 +-- .../java/be/nbb/sdmx/facade/xml/Sdmxml.java | 29 +++++++++ .../stream/XMLStreamCompactDataCursor.java | 14 ++-- .../stream/XMLStreamGenericDataCursor.java | 50 +++++++-------- .../xml/stream/XMLStreamStructure20.java | 51 ++++++++------- .../xml/stream/XMLStreamStructure21.java | 53 ++++++++------- .../xml/stream/XMLStreamStructure20Test.java | 64 +++++++++---------- .../xml/stream/XMLStreamStructure21Test.java | 40 ++++++------ 12 files changed, 223 insertions(+), 203 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index d7cbe7bc..1f37329f 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -17,14 +17,10 @@ package internal.file; import be.nbb.sdmx.facade.DataStructure; -import static internal.file.SdmxDecoder.DataType.COMPACT20; -import static internal.file.SdmxDecoder.DataType.COMPACT21; -import static internal.file.SdmxDecoder.DataType.GENERIC20; -import static internal.file.SdmxDecoder.DataType.GENERIC21; +import static internal.file.SdmxDecoder.DataType.*; import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; +import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.io.IOException; -import java.io.Reader; -import javax.xml.stream.XMLInputFactory; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; @@ -36,19 +32,18 @@ */ final class DataStructureDecoder { - private static final String NS_21 = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"; - - public static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, XMLInputFactory factory, Reader stream) throws IOException { - try { - XMLStreamReader reader = factory.createXMLStreamReader(stream); + public static XMLStream of(SdmxDecoder.DataType dataType) { + return reader -> { try { - return decodeDataStructure(dataType, reader); - } finally { - reader.close(); + try { + return decodeDataStructure(dataType, reader); + } finally { + reader.close(); + } + } catch (XMLStreamException ex) { + throw new IOException(ex); } - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + }; } private static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, XMLStreamReader reader) throws IOException, XMLStreamException { @@ -318,11 +313,8 @@ private static DataStructure compact21(XMLStreamReader reader) throws XMLStreamE switch (reader.next()) { case START_ELEMENT: switch (reader.getLocalName()) { - case "Structure": - if (reader.getName().getNamespaceURI().equals(NS_21)) { - builder.refId(reader.getAttributeValue(null, "structureID")); - builder.timeDimensionId(reader.getAttributeValue(null, "dimensionAtObservation")); - } + case "Header": + compact21Header(reader, builder); break; case "DataSet": compact21DataSet(reader, builder); @@ -334,6 +326,30 @@ private static DataStructure compact21(XMLStreamReader reader) throws XMLStreamE return builder.build(); } + private static void compact21Header(XMLStreamReader reader, CustomDataStructureBuilder builder) throws XMLStreamException { + while (reader.hasNext()) { + switch (reader.next()) { + case START_ELEMENT: + switch (reader.getLocalName()) { + case "Structure": + String structureId = reader.getAttributeValue(null, "structureID"); + String dimensionAtObservation = reader.getAttributeValue(null, "dimensionAtObservation"); + if (structureId != null && dimensionAtObservation != null) { + builder.refId(structureId); + builder.timeDimensionId(dimensionAtObservation); + } + break; + } + break; + case END_ELEMENT: + if (reader.getLocalName().equals("Header")) { + return; + } + break; + } + } + } + private static void compact21DataSet(XMLStreamReader reader, CustomDataStructureBuilder builder) throws XMLStreamException { while (reader.hasNext()) { switch (reader.next()) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java index 76e62c27..cbe4e046 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java @@ -16,14 +16,10 @@ */ package internal.file; -import static internal.file.SdmxDecoder.DataType.COMPACT20; -import static internal.file.SdmxDecoder.DataType.COMPACT21; -import static internal.file.SdmxDecoder.DataType.GENERIC20; -import static internal.file.SdmxDecoder.DataType.GENERIC21; -import static internal.file.SdmxDecoder.DataType.UNKNOWN; +import static be.nbb.sdmx.facade.xml.Sdmxml.*; +import be.nbb.sdmx.facade.xml.stream.XMLStream; +import static internal.file.SdmxDecoder.DataType.*; import java.io.IOException; -import java.io.Reader; -import javax.xml.stream.XMLInputFactory; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; @@ -35,21 +31,18 @@ */ final class DataTypeProbe { - private static final String NS_10 = "http://www.SDMX.org/resources/SDMXML/schemas/v1_0/message"; - private static final String NS_20 = "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message"; - private static final String NS_21 = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"; - - public static SdmxDecoder.DataType probeDataType(XMLInputFactory factory, Reader stream) throws IOException { - try { - XMLStreamReader reader = factory.createXMLStreamReader(stream); + public static XMLStream of() { + return reader -> { try { - return probeDataType(reader); - } finally { - reader.close(); + try { + return probeDataType(reader); + } finally { + reader.close(); + } + } catch (XMLStreamException ex) { + throw new IOException(ex); } - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + }; } private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws XMLStreamException { @@ -60,9 +53,9 @@ private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws level++; if (level == 2 && reader.getLocalName().equals("Header")) { switch (reader.getNamespaceURI()) { - case NS_10: + case NS_V10_URI: return UNKNOWN; - case NS_20: + case NS_V20_URI: while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: @@ -77,7 +70,7 @@ private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws } } return COMPACT20; - case NS_21: + case NS_V21_URI: while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index 3b42644d..ab7365db 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -29,8 +29,6 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.XMLStream; -import java.io.Reader; -import java.nio.file.Files; import java.util.List; /** @@ -54,9 +52,7 @@ public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOExcep } private DataType probeDataType(File data) throws IOException { - try (Reader stream = Files.newBufferedReader(data.toPath())) { - return DataTypeProbe.probeDataType(factory, stream); - } + return DataTypeProbe.of().get(factory, data.toPath(), StandardCharsets.UTF_8); } private DataStructure parseStruct(DataType dataType, File structure, LanguagePriorityList langs) throws IOException { @@ -77,8 +73,6 @@ private XMLStream> getStructSupplier(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - try (Reader stream = Files.newBufferedReader(data.toPath(), StandardCharsets.UTF_8)) { - return DataStructureDecoder.decodeDataStructure(dataType, factory, stream); - } + return DataStructureDecoder.of(dataType).get(factory, data.toPath(), StandardCharsets.UTF_8); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index 711e1828..0a6b0df3 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -45,7 +45,7 @@ public void testDecodeGeneric20() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC20, SdmxSource.XIF, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.GENERIC20).get(SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -64,7 +64,7 @@ public void testDecodeCompact20() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT20, SdmxSource.XIF, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.COMPACT20).get(SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -85,7 +85,7 @@ public void testDecodeGeneric21() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.GENERIC21, SdmxSource.XIF, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.GENERIC21).get(SdmxSource.XIF, stream)).isEqualTo(ds); } } @@ -106,7 +106,7 @@ public void testDecodeCompact21() throws IOException { .build(); try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataStructureDecoder.decodeDataStructure(SdmxDecoder.DataType.COMPACT21, SdmxSource.XIF, stream)).isEqualTo(ds); + assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.COMPACT21).get(SdmxSource.XIF, stream)).isEqualTo(ds); } } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java index 09450b2c..9a79e009 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java @@ -31,28 +31,28 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC20); + assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC20); } } @Test public void testDecodeCompact20() throws IOException { try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT20); + assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT20); } } @Test public void testDecodeGeneric21() throws IOException { try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC21); + assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC21); } } @Test public void testDecodeCompact21() throws IOException { try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataTypeProbe.probeDataType(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT21); + assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT21); } } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java new file mode 100644 index 00000000..7386cc8f --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class Sdmxml { + + public static final String NS_V10_URI = "http://www.SDMX.org/resources/SDMXML/schemas/v1_0/message"; + public static final String NS_V20_URI = "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message"; + public static final String NS_V21_URI = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"; +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 8d7da425..dbe73669 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -37,9 +37,9 @@ */ final class XMLStreamCompactDataCursor implements DataCursor { - private static final String DATASET_ELEMENT = "DataSet"; - private static final String SERIES_ELEMENT = "Series"; - private static final String OBS_ELEMENT = "Obs"; + private static final String DATASET_TAG = "DataSet"; + private static final String SERIES_TAG = "Series"; + private static final String OBS_TAG = "Obs"; private final XMLStreamReader reader; private final Key.Builder keyBuilder; @@ -159,9 +159,9 @@ private void checkObsState() throws IOException, IllegalStateException { private Status onDataSet(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(SERIES_ELEMENT) ? parseSeries() : CONTINUE; + return localName.equals(SERIES_TAG) ? parseSeries() : CONTINUE; } else { - return localName.equals(DATASET_ELEMENT) ? HALT : CONTINUE; + return localName.equals(DATASET_TAG) ? HALT : CONTINUE; } } @@ -184,9 +184,9 @@ private void parserSerieHead() { private Status onSeriesBody(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(OBS_ELEMENT) ? parseObs() : CONTINUE; + return localName.equals(OBS_TAG) ? parseObs() : CONTINUE; } else { - return localName.equals(SERIES_ELEMENT) ? HALT : CONTINUE; + return localName.equals(SERIES_TAG) ? HALT : CONTINUE; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 90f1c788..4b1d4b3d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -48,14 +48,14 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Key.Builder key return new XMLStreamGenericDataCursor(reader, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX21); } - private static final String DATASET_ELEMENT = "DataSet"; - private static final String SERIES_ELEMENT = "Series"; - private static final String OBS_ELEMENT = "Obs"; - private static final String OBS_VALUE_ELEMENT = "ObsValue"; - private static final String SERIES_KEY_ELEMENT = "SeriesKey"; - private static final String ATTRIBUTES_ELEMENT = "Attributes"; - private static final String VALUE_ELEMENT = "Value"; - private static final String VALUE_ATTRIBUTE = "value"; + private static final String DATASET_TAG = "DataSet"; + private static final String SERIES_TAG = "Series"; + private static final String OBS_TAG = "Obs"; + private static final String OBS_VALUE_TAG = "ObsValue"; + private static final String SERIES_KEY_TAG = "SeriesKey"; + private static final String ATTRIBUTES_TAG = "Attributes"; + private static final String VALUE_TAG = "Value"; + private static final String VALUE_ATTR = "value"; private final XMLStreamReader reader; private final Key.Builder keyBuilder; @@ -180,9 +180,9 @@ private void checkObsState() throws IOException, IllegalStateException { private Status onDataSet(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(SERIES_ELEMENT) ? parseSeries() : CONTINUE; + return localName.equals(SERIES_TAG) ? parseSeries() : CONTINUE; } else { - return localName.equals(DATASET_ELEMENT) ? HALT : CONTINUE; + return localName.equals(DATASET_TAG) ? HALT : CONTINUE; } } @@ -195,17 +195,17 @@ private Status parseSeries() throws XMLStreamException { private Status onSeriesHead(boolean start, String localName) throws XMLStreamException { if (start) { switch (localName) { - case SERIES_KEY_ELEMENT: + case SERIES_KEY_TAG: return parseSeriesKey(); - case ATTRIBUTES_ELEMENT: + case ATTRIBUTES_TAG: return parseAttributes(); - case OBS_ELEMENT: + case OBS_TAG: return HALT; default: return CONTINUE; } } else { - return localName.equals(SERIES_ELEMENT) ? HALT : CONTINUE; + return localName.equals(SERIES_TAG) ? HALT : CONTINUE; } } @@ -221,17 +221,17 @@ private Status parseAttributes() throws XMLStreamException { private Status onSeriesKey(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(VALUE_ELEMENT) ? parseSeriesKeyValue() : CONTINUE; + return localName.equals(VALUE_TAG) ? parseSeriesKeyValue() : CONTINUE; } else { - return localName.equals(SERIES_KEY_ELEMENT) ? HALT : CONTINUE; + return localName.equals(SERIES_KEY_TAG) ? HALT : CONTINUE; } } private Status onAttributes(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(VALUE_ELEMENT) ? parseAttributesValue() : CONTINUE; + return localName.equals(VALUE_TAG) ? parseAttributesValue() : CONTINUE; } else { - return localName.equals(ATTRIBUTES_ELEMENT) ? HALT : CONTINUE; + return localName.equals(ATTRIBUTES_TAG) ? HALT : CONTINUE; } } @@ -246,18 +246,18 @@ private Status parseAttributesValue() throws XMLStreamException { } private boolean isCurrentElementStartOfObs() { - return reader.getEventType() == XMLStreamReader.START_ELEMENT && OBS_ELEMENT.equals(reader.getLocalName()); + return reader.getEventType() == XMLStreamReader.START_ELEMENT && OBS_TAG.equals(reader.getLocalName()); } private boolean isCurrentElementEnfOfSeries() { - return reader.getEventType() == XMLStreamReader.END_ELEMENT && SERIES_ELEMENT.equals(reader.getLocalName()); + return reader.getEventType() == XMLStreamReader.END_ELEMENT && SERIES_TAG.equals(reader.getLocalName()); } private Status onSeriesBody(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(OBS_ELEMENT) ? parseObs() : CONTINUE; + return localName.equals(OBS_TAG) ? parseObs() : CONTINUE; } else { - return localName.equals(SERIES_ELEMENT) ? HALT : CONTINUE; + return localName.equals(SERIES_TAG) ? HALT : CONTINUE; } } @@ -266,9 +266,9 @@ private Status onObs(boolean start, String localName) throws XMLStreamException if (localName.equals(headParser.getTimeELement())) { return parseObsTime(); } - return localName.equals(OBS_VALUE_ELEMENT) ? parseObsValue() : CONTINUE; + return localName.equals(OBS_VALUE_TAG) ? parseObsValue() : CONTINUE; } else { - return localName.equals(OBS_ELEMENT) ? HALT : CONTINUE; + return localName.equals(OBS_TAG) ? HALT : CONTINUE; } } @@ -283,7 +283,7 @@ private Status parseObsTime() throws XMLStreamException { } private Status parseObsValue() { - obsParser.value(reader.getAttributeValue(null, VALUE_ATTRIBUTE)); + obsParser.value(reader.getAttributeValue(null, VALUE_ATTR)); return CONTINUE; } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index a9fb1488..4715ee48 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -30,6 +30,7 @@ import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTags; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; +import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V20_URI; import javax.annotation.Nonnull; /** @@ -38,6 +39,29 @@ */ final class XMLStreamStructure20 { + private static final String HEADER_TAG = "Header"; + private static final String CODE_LISTS_TAG = "CodeLists"; + private static final String CONCEPTS_TAG = "Concepts"; + private static final String KEY_FAMILIES_TAG = "KeyFamilies"; + private static final String CODE_LIST_TAG = "CodeList"; + private static final String CONCEPT_TAG = "Concept"; + private static final String KEY_FAMILY_TAG = "KeyFamily"; + private static final String CODE_TAG = "Code"; + private static final String DESCRIPTION_TAG = "Description"; + private static final String NAME_TAG = "Name"; + private static final String COMPONENTS_TAG = "Components"; + private static final String DIMENSION_TAG = "Dimension"; + private static final String TIME_DIMENSION_TAG = "TimeDimension"; + private static final String PRIMARY_MEASURE_TAG = "PrimaryMeasure"; + + private static final String ID_ATTR = "id"; + private static final String AGENCY_ID_ATTR = "agencyID"; + private static final String VERSION_ATTR = "version"; + private static final String LANG_ATTR = "lang"; + private static final String VALUE_ATTR = "value"; + private static final String CONCEPT_REF_ATTR = "conceptRef"; + private static final String CODELIST_ATTR = "codelist"; + private final TextBuilder structureLabel; private final TextBuilder label; @@ -70,34 +94,9 @@ public List parse(@Nonnull XMLStreamReader reader) throws XMLStre return result; } - private static final String NS_20 = "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message"; - - private static final String HEADER_TAG = "Header"; - private static final String CODE_LISTS_TAG = "CodeLists"; - private static final String CONCEPTS_TAG = "Concepts"; - private static final String KEY_FAMILIES_TAG = "KeyFamilies"; - private static final String CODE_LIST_TAG = "CodeList"; - private static final String CONCEPT_TAG = "Concept"; - private static final String KEY_FAMILY_TAG = "KeyFamily"; - private static final String CODE_TAG = "Code"; - private static final String DESCRIPTION_TAG = "Description"; - private static final String NAME_TAG = "Name"; - private static final String COMPONENTS_TAG = "Components"; - private static final String DIMENSION_TAG = "Dimension"; - private static final String TIME_DIMENSION_TAG = "TimeDimension"; - private static final String PRIMARY_MEASURE_TAG = "PrimaryMeasure"; - - private static final String ID_ATTR = "id"; - private static final String AGENCY_ID_ATTR = "agencyID"; - private static final String VERSION_ATTR = "version"; - private static final String LANG_ATTR = "lang"; - private static final String VALUE_ATTR = "value"; - private static final String CONCEPT_REF_ATTR = "conceptRef"; - private static final String CODELIST_ATTR = "codelist"; - private void parseHeader(XMLStreamReader reader) throws XMLStreamException { String ns = reader.getNamespaceURI(); - check(NS_20.equals(ns), reader, "Invalid namespace '%s'", ns); + check(NS_V20_URI.equals(ns), reader, "Invalid namespace '%s'", ns); } private void parseCodelists(XMLStreamReader reader, Map> codelists) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 4228e549..97f4fed1 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -30,6 +30,7 @@ import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTags; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; +import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; import javax.annotation.Nonnull; /** @@ -38,32 +39,6 @@ */ final class XMLStreamStructure21 { - private final TextBuilder structureLabel; - private final TextBuilder label; - - XMLStreamStructure21(LanguagePriorityList languages) { - this.structureLabel = new TextBuilder(languages); - this.label = new TextBuilder(languages); - } - - @Nonnull - public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { - List result = new ArrayList<>(); - while (nextTags(reader, "")) { - switch (reader.getLocalName()) { - case HEADER_TAG: - parseHeader(reader); - break; - case STRUCTURES_TAG: - parseStructures(reader, result); - break; - } - } - return result; - } - - private static final String NS_21 = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"; - private static final String HEADER_TAG = "Header"; private static final String STRUCTURES_TAG = "Structures"; private static final String CODELISTS_TAG = "Codelists"; @@ -89,9 +64,33 @@ public List parse(@Nonnull XMLStreamReader reader) throws XMLStre private static final String VERSION_ATTR = "version"; private static final String LANG_ATTR = "lang"; + private final TextBuilder structureLabel; + private final TextBuilder label; + + XMLStreamStructure21(LanguagePriorityList languages) { + this.structureLabel = new TextBuilder(languages); + this.label = new TextBuilder(languages); + } + + @Nonnull + public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { + List result = new ArrayList<>(); + while (nextTags(reader, "")) { + switch (reader.getLocalName()) { + case HEADER_TAG: + parseHeader(reader); + break; + case STRUCTURES_TAG: + parseStructures(reader, result); + break; + } + } + return result; + } + private void parseHeader(XMLStreamReader reader) throws XMLStreamException { String ns = reader.getNamespaceURI(); - check(NS_21.equals(ns), reader, "Invalid namespace '%s'", ns); + check(NS_V21_URI.equals(ns), reader, "Invalid namespace '%s'", ns); } private void parseStructures(XMLStreamReader reader, List structs) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index a87132ae..15e9d5db 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -16,10 +16,12 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; -import java.io.InputStreamReader; +import java.io.IOException; +import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; @@ -37,45 +39,37 @@ public class XMLStreamStructure20Test { public void test() throws Exception { XMLInputFactory factory = XMLInputFactory.newFactory(); - XMLStreamStructure20 p1 = new XMLStreamStructure20(LanguagePriorityList.ANY); + XMLStream> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - try (InputStreamReader stream = SdmxSource.NBB_DATA_STRUCTURE.openReader()) { - assertThat(p1.parse(factory.createXMLStreamReader(stream))).hasSize(1).element(0).satisfies(o -> { - assertThat(o.getLabel()).isEqualTo("My first dataset"); - assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); - assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); - assertThat(o.getRef()).isEqualTo(DataStructureRef.of("NBB", "TEST_DATASET", null)); - assertThat(o.getDimensions()).hasSize(3).element(0).satisfies(x -> { - assertThat(x.getId()).isEqualTo("SUBJECT"); - assertThat(x.getLabel()).isEqualTo("Subject"); - assertThat(x.getPosition()).isEqualTo(1); - }); + assertThat(p1.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(o.getLabel()).isEqualTo("My first dataset"); + assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); + assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); + assertThat(o.getRef()).isEqualTo(DataStructureRef.of("NBB", "TEST_DATASET", null)); + assertThat(o.getDimensions()).hasSize(3).element(0).satisfies(x -> { + assertThat(x.getId()).isEqualTo("SUBJECT"); + assertThat(x.getLabel()).isEqualTo("Subject"); + assertThat(x.getPosition()).isEqualTo(1); }); - } + }); - XMLStreamStructure20 p2 = new XMLStreamStructure20(LanguagePriorityList.parse("fr")); + XMLStream> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - try (InputStreamReader stream = SdmxSource.NBB_DATA_STRUCTURE.openReader()) { - assertThat(p2.parse(factory.createXMLStreamReader(stream))).hasSize(1).element(0).satisfies(o -> { - assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); - assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); - assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); - assertThat(o.getRef()).isEqualTo(DataStructureRef.of("NBB", "TEST_DATASET", null)); - assertThat(o.getDimensions()).hasSize(3).element(0).satisfies(x -> { - assertThat(x.getId()).isEqualTo("SUBJECT"); - assertThat(x.getLabel()).isEqualTo("Sujet"); - assertThat(x.getPosition()).isEqualTo(1); - }); + assertThat(p2.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); + assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); + assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); + assertThat(o.getRef()).isEqualTo(DataStructureRef.of("NBB", "TEST_DATASET", null)); + assertThat(o.getDimensions()).hasSize(3).element(0).satisfies(x -> { + assertThat(x.getId()).isEqualTo("SUBJECT"); + assertThat(x.getLabel()).isEqualTo("Sujet"); + assertThat(x.getPosition()).isEqualTo(1); }); - } + }); - try (InputStreamReader stream = SdmxSource.ECB_DATA_STRUCTURE.openReader()) { - assertThatThrownBy(() -> p1.parse(factory.createXMLStreamReader(stream))) - .isInstanceOf(XMLStreamException.class) - .hasMessageContaining("Invalid namespace") - .hasNoCause(); - } - - assertThatThrownBy(() -> p1.parse(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> p1.get(factory, SdmxSource.ECB_DATA_STRUCTURE.openReader())) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf(XMLStreamException.class) + .hasMessageContaining("Invalid namespace"); } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index 4296c075..438f1251 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -16,10 +16,12 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; -import java.io.InputStreamReader; +import java.io.IOException; +import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; @@ -37,29 +39,23 @@ public class XMLStreamStructure21Test { public void test() throws Exception { XMLInputFactory factory = XMLInputFactory.newFactory(); - XMLStreamStructure21 parser = new XMLStreamStructure21(LanguagePriorityList.ANY); + XMLStream> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - try (InputStreamReader stream = SdmxSource.ECB_DATA_STRUCTURE.openReader()) { - assertThat(parser.parse(factory.createXMLStreamReader(stream))).hasSize(1).element(0).satisfies(o -> { - assertThat(o.getLabel()).isEqualTo("AMECO"); - assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); - assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); - assertThat(o.getRef()).isEqualTo(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); - assertThat(o.getDimensions()).hasSize(7).element(0).satisfies(x -> { - assertThat(x.getId()).isEqualTo("FREQ"); - assertThat(x.getLabel()).isEqualTo("Frequency"); - assertThat(x.getPosition()).isEqualTo(1); - }); + assertThat(parser.get(factory, SdmxSource.ECB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(o.getLabel()).isEqualTo("AMECO"); + assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); + assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); + assertThat(o.getRef()).isEqualTo(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); + assertThat(o.getDimensions()).hasSize(7).element(0).satisfies(x -> { + assertThat(x.getId()).isEqualTo("FREQ"); + assertThat(x.getLabel()).isEqualTo("Frequency"); + assertThat(x.getPosition()).isEqualTo(1); }); - } + }); - try (InputStreamReader stream = SdmxSource.NBB_DATA_STRUCTURE.openReader()) { - assertThatThrownBy(() -> parser.parse(factory.createXMLStreamReader(stream))) - .isInstanceOf(XMLStreamException.class) - .hasMessageContaining("Invalid namespace") - .hasNoCause(); - } - - assertThatThrownBy(() -> parser.parse(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> parser.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf(XMLStreamException.class) + .hasMessageContaining("Invalid namespace"); } } From 7b8f9b1bd18e5265160336a7c839977c85cd3374 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 15 Nov 2017 11:59:07 +0100 Subject: [PATCH 35/68] Fixed xml resource leak. --- .../internal/connectors/TestResource.java | 6 +- .../internal/file/DataStructureDecoder.java | 17 +- .../java/internal/file/DataTypeProbe.java | 13 +- .../internal/file/FileSdmxConnection.java | 2 +- .../internal/file/XMLStreamSdmxDecoder.java | 6 +- .../file/DataStructureDecoderTest.java | 29 ++-- .../java/internal/file/DataTypeProbeTest.java | 21 +-- .../nbb/sdmx/facade/samples/ByteSource.java | 14 -- .../main/java/be/nbb/sdmx/facade/util/IO.java | 2 +- .../facade/xml/stream/SdmxXmlStreams.java | 24 +-- .../be/nbb/sdmx/facade/xml/stream/XFunc.java | 30 ++++ .../nbb/sdmx/facade/xml/stream/XMLStream.java | 43 ++++- .../stream/XMLStreamCompactDataCursor.java | 13 +- .../stream/XMLStreamGenericDataCursor.java | 21 ++- .../sdmx/facade/xml/stream/XMLStreamUtil.java | 41 ++--- .../sdmx/facade/util/SeriesSupportTest.java | 8 +- .../XMLStreamCompactDataCursorTest.java | 17 +- .../XMLStreamGenericDataCursorTest.java | 25 ++- .../xml/stream/XMLStreamStructure20Test.java | 9 +- .../xml/stream/XMLStreamStructure21Test.java | 7 +- .../sdmx/facade/xml/stream/XMLStreamTest.java | 150 ++++++++++++++++++ .../connectors/drivers/InseeDriver.java | 2 +- .../connectors/drivers/Sdmx21Driver.java | 2 +- .../connectors/ConnectorsResource.java | 7 +- .../internal/connectors/FacadeResource.java | 17 +- 25 files changed, 340 insertions(+), 186 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 663bb39b..33b9ca1f 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -58,7 +58,7 @@ public static final SdmxRepository nbb() { .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dsd = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); - try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).get(SdmxSource.XIF, SdmxSource.NBB_DATA.openReader())) { + try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA::openReader)) { result.copyOf(flowRef, cursor); } return result @@ -83,7 +83,7 @@ public static final SdmxRepository ecb() { .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dfs = dataStructures.get(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); - try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).get(SdmxSource.XIF, SdmxSource.ECB_DATA.openReader())) { + try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { result.copyOf(flowRef, cursor); } return result @@ -111,7 +111,7 @@ private static Dataflow toDataFlow(DataFlowStructure input) { private static T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { XMLEventReader r = null; try { - r = xml.openXmlEvent(SdmxSource.XIF); + r = SdmxSource.XIF.createXMLEventReader(xml.openReader()); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index 1f37329f..63cb236d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -20,7 +20,6 @@ import static internal.file.SdmxDecoder.DataType.*; import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; import be.nbb.sdmx.facade.xml.stream.XMLStream; -import java.io.IOException; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; @@ -33,20 +32,10 @@ final class DataStructureDecoder { public static XMLStream of(SdmxDecoder.DataType dataType) { - return reader -> { - try { - try { - return decodeDataStructure(dataType, reader); - } finally { - reader.close(); - } - } catch (XMLStreamException ex) { - throw new IOException(ex); - } - }; + return XMLStream.of(o -> decodeDataStructure(dataType, o)); } - private static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, XMLStreamReader reader) throws IOException, XMLStreamException { + private static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, XMLStreamReader reader) throws XMLStreamException { switch (dataType) { case GENERIC20: return generic20(reader); @@ -57,7 +46,7 @@ private static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, case COMPACT21: return compact21(reader); default: - throw new IOException("Don't know how to handle '" + dataType + "'"); + throw new XMLStreamException("Don't know how to handle '" + dataType + "'"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java index cbe4e046..da47ee2a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java @@ -19,7 +19,6 @@ import static be.nbb.sdmx.facade.xml.Sdmxml.*; import be.nbb.sdmx.facade.xml.stream.XMLStream; import static internal.file.SdmxDecoder.DataType.*; -import java.io.IOException; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; @@ -32,17 +31,7 @@ final class DataTypeProbe { public static XMLStream of() { - return reader -> { - try { - try { - return probeDataType(reader); - } finally { - reader.close(); - } - } catch (XMLStreamException ex) { - throw new IOException(ex); - } - }; + return XMLStream.of(DataTypeProbe::probeDataType); } private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 8e9dd5e3..a1aaa596 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -120,7 +120,7 @@ protected SdmxDecoder.Info decode() throws IOException { } protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return getDataSupplier(entry.getDataType(), entry.getDataStructure()).get(factory, files.getData().toPath(), StandardCharsets.UTF_8); + return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factory, files.getData().toPath(), StandardCharsets.UTF_8); } private XMLStream getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index ab7365db..196a149d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -52,11 +52,11 @@ public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOExcep } private DataType probeDataType(File data) throws IOException { - return DataTypeProbe.of().get(factory, data.toPath(), StandardCharsets.UTF_8); + return DataTypeProbe.of().parseFile(factory, data.toPath(), StandardCharsets.UTF_8); } private DataStructure parseStruct(DataType dataType, File structure, LanguagePriorityList langs) throws IOException { - return getStructSupplier(dataType, langs).get(factory, structure.toPath(), StandardCharsets.UTF_8).get(0); + return getStructSupplier(dataType, langs).parseFile(factory, structure.toPath(), StandardCharsets.UTF_8).get(0); } private XMLStream> getStructSupplier(DataType o, LanguagePriorityList langs) throws IOException { @@ -73,6 +73,6 @@ private XMLStream> getStructSupplier(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - return DataStructureDecoder.of(dataType).get(factory, data.toPath(), StandardCharsets.UTF_8); + return DataStructureDecoder.of(dataType).parseFile(factory, data.toPath(), StandardCharsets.UTF_8); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index 0a6b0df3..bd8a6a37 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -23,7 +23,6 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import static org.assertj.core.api.Assertions.assertThat; import static internal.file.CustomDataStructureBuilder.dimension; -import java.io.Reader; /** * @@ -44,9 +43,10 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.GENERIC20).get(SdmxSource.XIF, stream)).isEqualTo(ds); - } + assertThat(DataStructureDecoder + .of(SdmxDecoder.DataType.GENERIC20) + .parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader) + ).isEqualTo(ds); } @Test @@ -63,9 +63,10 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.COMPACT20).get(SdmxSource.XIF, stream)).isEqualTo(ds); - } + assertThat(DataStructureDecoder + .of(SdmxDecoder.DataType.COMPACT20) + .parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader) + ).isEqualTo(ds); } @Test @@ -84,9 +85,10 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.GENERIC21).get(SdmxSource.XIF, stream)).isEqualTo(ds); - } + assertThat(DataStructureDecoder + .of(SdmxDecoder.DataType.GENERIC21) + .parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader) + ).isEqualTo(ds); } @Test @@ -105,8 +107,9 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataStructureDecoder.of(SdmxDecoder.DataType.COMPACT21).get(SdmxSource.XIF, stream)).isEqualTo(ds); - } + assertThat(DataStructureDecoder + .of(SdmxDecoder.DataType.COMPACT21) + .parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader) + ).isEqualTo(ds); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java index 9a79e009..3379dbd0 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java @@ -19,7 +19,6 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import org.junit.Test; -import java.io.Reader; import static org.assertj.core.api.Assertions.assertThat; /** @@ -30,29 +29,25 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { - try (Reader stream = SdmxSource.OTHER_GENERIC20.openReader()) { - assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC20); - } + assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader)) + .isEqualTo(SdmxDecoder.DataType.GENERIC20); } @Test public void testDecodeCompact20() throws IOException { - try (Reader stream = SdmxSource.OTHER_COMPACT20.openReader()) { - assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT20); - } + assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader)) + .isEqualTo(SdmxDecoder.DataType.COMPACT20); } @Test public void testDecodeGeneric21() throws IOException { - try (Reader stream = SdmxSource.OTHER_GENERIC21.openReader()) { - assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.GENERIC21); - } + assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader)) + .isEqualTo(SdmxDecoder.DataType.GENERIC21); } @Test public void testDecodeCompact21() throws IOException { - try (Reader stream = SdmxSource.OTHER_COMPACT21.openReader()) { - assertThat(DataTypeProbe.of().get(SdmxSource.XIF, stream)).isEqualTo(SdmxDecoder.DataType.COMPACT21); - } + assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader)) + .isEqualTo(SdmxDecoder.DataType.COMPACT21); } } diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java index 78a230ea..23ea597d 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/ByteSource.java @@ -26,10 +26,6 @@ import java.nio.file.StandardCopyOption; import java.util.Objects; import javax.annotation.Nonnull; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; /** * @@ -40,16 +36,6 @@ public interface ByteSource { @Nonnull InputStream openStream() throws IOException; - @Nonnull - default XMLStreamReader openXmlStream(@Nonnull XMLInputFactory f) throws XMLStreamException, IOException { - return f.createXMLStreamReader(openReader()); - } - - @Nonnull - default XMLEventReader openXmlEvent(@Nonnull XMLInputFactory f) throws XMLStreamException, IOException { - return f.createXMLEventReader(openReader()); - } - @Nonnull default InputStreamReader openReader() throws IOException { return new InputStreamReader(openStream(), StandardCharsets.UTF_8); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java index 9aa1aaa5..31820e8c 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java @@ -98,7 +98,7 @@ public T next() { iter, Spliterator.ORDERED | Spliterator.NONNULL), false); } - private static Runnable asUncheckedRunnable(Closeable c) { + private Runnable asUncheckedRunnable(Closeable c) { return () -> { try { c.close(); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index 72fbd26c..d361a808 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -24,9 +24,6 @@ import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.ReaderFunc; /** * @@ -42,7 +39,7 @@ public XMLStream compactData20(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> new XMLStreamCompactDataCursor(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); + return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); } @Nonnull @@ -52,7 +49,7 @@ public XMLStream compactData21(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> new XMLStreamCompactDataCursor(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull @@ -62,7 +59,7 @@ public XMLStream genericData20(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> XMLStreamGenericDataCursor.sdmx20(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return (o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull @@ -72,25 +69,16 @@ public XMLStream genericData21(@Nonnull DataStructure dsd) throws IO @Nonnull public XMLStream genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return o -> XMLStreamGenericDataCursor.sdmx21(o, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return (o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull public XMLStream> struct20(@Nonnull LanguagePriorityList langs) throws IOException { - return o -> struct(o, new XMLStreamStructure20(langs)::parse); + return XMLStream.of(new XMLStreamStructure20(langs)::parse); } @Nonnull public XMLStream> struct21(@Nonnull LanguagePriorityList langs) throws IOException { - return o -> struct(o, new XMLStreamStructure21(langs)::parse); - } - - @Nonnull - private List struct(@Nonnull XMLStreamReader reader, @Nonnull ReaderFunc> func) throws IOException { - try { - return XMLStreamUtil.with(reader, func); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + return XMLStream.of(new XMLStreamStructure21(langs)::parse); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java new file mode 100644 index 00000000..4c14cad7 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * + * @author Philippe Charles + * @param + */ +public interface XFunc { + + T apply(XMLStreamReader reader) throws XMLStreamException; +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java index 6a0cdb54..185f422d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java @@ -16,7 +16,10 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.sdmx.facade.util.IO; +import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.io.Reader; import java.nio.charset.Charset; import java.nio.file.Files; @@ -34,23 +37,47 @@ public interface XMLStream { @Nonnull - default T get(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { - return get(xf, Files.newBufferedReader(path, cs)); + default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { + return parseReader(xf, () -> Files.newBufferedReader(path, cs)); } @Nonnull - default T get(@Nonnull XMLInputFactory xf, @Nonnull Reader stream) throws IOException { - XMLStreamReader reader; + default T parseStream(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier supplier, @Nonnull Charset cs) throws IOException { + InputStream stream = supplier.getWithIO(); try { - reader = xf.createXMLStreamReader(stream); + XMLStreamReader xml = xf.createXMLStreamReader(stream, cs.name()); + return parse(xml, () -> XMLStreamUtil.closeBoth(xml, stream)); } catch (XMLStreamException ex) { - throw new IOException("Failed to create XML reader", ex); + XMLStreamUtil.ensureClosed(ex, stream); + throw new IOException("Failed to create XMLStreamReader from a stream", ex); } + } + + @Nonnull + default T parseReader(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier supplier) throws IOException { + Reader reader = supplier.getWithIO(); - return get(reader); + try { + XMLStreamReader xml = xf.createXMLStreamReader(reader); + return parse(xml, () -> XMLStreamUtil.closeBoth(xml, reader)); + } catch (XMLStreamException ex) { + XMLStreamUtil.ensureClosed(ex, reader); + throw new IOException("Failed to create XMLStreamReader from a reader", ex); + } } @Nonnull - T get(@Nonnull XMLStreamReader reader) throws IOException; + T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; + + @Nonnull + static XMLStream of(@Nonnull XFunc func) { + return (reader, onClose) -> { + try (Closeable c = onClose) { + return func.apply(reader); + } catch (XMLStreamException ex) { + throw new IOException(ex); + } + }; + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index dbe73669..2ce700a8 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -30,6 +30,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.util.FreqParser; +import java.io.Closeable; /** * @@ -42,6 +43,7 @@ final class XMLStreamCompactDataCursor implements DataCursor { private static final String OBS_TAG = "Obs"; private final XMLStreamReader reader; + private final Closeable onClose; private final Key.Builder keyBuilder; private final AttributesBuilder attributesBuilder; private final ObsParser obsParser; @@ -52,8 +54,9 @@ final class XMLStreamCompactDataCursor implements DataCursor { private boolean hasSeries; private boolean hasObs; - XMLStreamCompactDataCursor(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, String timeDimensionId, String primaryMeasureId) { + XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, String timeDimensionId, String primaryMeasureId) { this.reader = reader; + this.onClose = onClose; this.keyBuilder = keyBuilder; this.attributesBuilder = new AttributesBuilder(); this.obsParser = obsParser; @@ -130,11 +133,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - try { - reader.close(); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + XMLStreamUtil.closeBoth(reader, onClose); } private void checkState() throws IOException { @@ -196,7 +195,7 @@ private Status parseObs() { return SUSPEND; } - private boolean nextWhile(XMLStreamUtil.Func func) throws XMLStreamException { + private boolean nextWhile(XMLStreamUtil.TagVisitor func) throws XMLStreamException { return XMLStreamUtil.nextWhile(reader, func); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 4b1d4b3d..1ff5eea2 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -30,6 +30,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.util.FreqParser; +import java.io.Closeable; import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nonnull; @@ -40,12 +41,12 @@ */ final class XMLStreamGenericDataCursor implements DataCursor { - static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { - return new XMLStreamGenericDataCursor(reader, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX20); + static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + return new XMLStreamGenericDataCursor(reader, onClose, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX20); } - static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { - return new XMLStreamGenericDataCursor(reader, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX21); + static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + return new XMLStreamGenericDataCursor(reader, onClose, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX21); } private static final String DATASET_TAG = "DataSet"; @@ -58,6 +59,7 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Key.Builder key private static final String VALUE_ATTR = "value"; private final XMLStreamReader reader; + private final Closeable onClose; private final Key.Builder keyBuilder; private final AttributesBuilder attributesBuilder; private final ObsParser obsParser; @@ -67,8 +69,9 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Key.Builder key private boolean hasSeries; private boolean hasObs; - private XMLStreamGenericDataCursor(XMLStreamReader reader, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { + private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { this.reader = reader; + this.onClose = onClose; this.keyBuilder = keyBuilder; this.attributesBuilder = new AttributesBuilder(); this.obsParser = obsParser; @@ -151,11 +154,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - try { - reader.close(); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } + XMLStreamUtil.closeBoth(reader, onClose); } private void checkState() throws IOException { @@ -287,7 +286,7 @@ private Status parseObsValue() { return CONTINUE; } - private boolean nextWhile(XMLStreamUtil.Func func) throws XMLStreamException { + private boolean nextWhile(XMLStreamUtil.TagVisitor func) throws XMLStreamException { return XMLStreamUtil.nextWhile(reader, func); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java index d62afa8c..504e82f6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java @@ -16,6 +16,8 @@ */ package be.nbb.sdmx.facade.xml.stream; +import java.io.Closeable; +import java.io.IOException; import java.util.function.Supplier; import javax.xml.stream.Location; import javax.xml.stream.XMLStreamException; @@ -32,12 +34,12 @@ enum Status { HALT, CONTINUE, SUSPEND; } - interface Func { + interface TagVisitor { Status visitTag(boolean start, String localName) throws XMLStreamException; } - static boolean nextWhile(XMLStreamReader reader, Func func) throws XMLStreamException { + static boolean nextWhile(XMLStreamReader reader, TagVisitor func) throws XMLStreamException { while (reader.hasNext()) { int event = reader.next(); if (event == XMLStreamReader.START_ELEMENT) { @@ -106,34 +108,21 @@ static void check(boolean expression, Supplier location, String messag } } - static int toInt(String input, int defaultValue) { - if (input != null) { - try { - return Integer.parseInt(input); - } catch (NumberFormatException ex) { - } - } - return defaultValue; - } - - static T with(XMLStreamReader reader, ReaderFunc func) throws XMLStreamException { - T result; + void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { try { - result = func.apply(reader); + reader.close(); } catch (XMLStreamException ex) { - try { - reader.close(); - } catch (XMLStreamException suppressed) { - ex.addSuppressed(suppressed); - } - throw (ex); + ensureClosed(ex, onClose); + throw new IOException("Failed to close xml stream reader", ex); } - reader.close(); - return result; + onClose.close(); } - interface ReaderFunc { - - T apply(XMLStreamReader reader) throws XMLStreamException; + void ensureClosed(XMLStreamException ex, Closeable onClose) throws IOException { + try { + onClose.close(); + } catch (IOException suppressed) { + ex.addSuppressed(suppressed); + } } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 5ccde929..5b030659 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -75,8 +75,8 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } } @@ -98,8 +98,8 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).get(SdmxSource.ECB_DATA_STRUCTURE.openXmlStream(SdmxSource.XIF)); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).get(SdmxSource.ECB_DATA.openXmlStream(SdmxSource.XIF))) { + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(120); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index a1d92a7f..4120de1a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; import be.nbb.sdmx.facade.util.FreqParser; +import java.io.InputStream; /** * @@ -39,9 +40,13 @@ public void testCompactData20() throws Exception { ByteSource xml = SdmxSource.OTHER_COMPACT20; Key.Builder builder = Key.builder("FREQ", "COLLECTION", "VIS_CTY", "JD_TYPE", "JD_CATEGORY"); - DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")); + DataCursorAssert.assertCompliance(() -> { + InputStream stream = xml.openStream(); + return new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE"); + }); - try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { + try (InputStream stream = xml.openStream(); + DataCursor o = new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -79,9 +84,13 @@ public void testCompactData21() throws Exception { ByteSource xml = SdmxSource.OTHER_COMPACT21; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")); + DataCursorAssert.assertCompliance(() -> { + InputStream stream = xml.openStream(); + return new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE"); + }); - try (DataCursor o = new XMLStreamCompactDataCursor(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { + try (InputStream stream = xml.openStream(); + DataCursor o = new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 50b34a00..183c78b7 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; import be.nbb.sdmx.facade.util.FreqParser; +import java.io.InputStream; /** * @@ -39,9 +40,13 @@ public void testGenericData20() throws Exception { ByteSource xml = SdmxSource.NBB_DATA; Key.Builder builder = Key.builder("SUBJECT", "LOCATION", "FREQUENCY"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20())); + DataCursorAssert.assertCompliance(() -> { + InputStream stream = xml.openStream(); + return XMLStreamGenericDataCursor.sdmx20(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20()); + }); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx20(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx20())) { + try (InputStream stream = xml.openStream(); + DataCursor o = XMLStreamGenericDataCursor.sdmx20(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20())) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -83,9 +88,13 @@ public void testGenericData21() throws Exception { ByteSource xml = SdmxSource.OTHER_GENERIC21; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))); + DataCursorAssert.assertCompliance(() -> { + InputStream stream = xml.openStream(); + return XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + }); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + try (InputStream stream = xml.openStream(); + DataCursor o = XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -119,9 +128,13 @@ public void testGenericData21Bis() throws Exception { ByteSource xml = SdmxSource.ECB_DATA; Key.Builder builder = Key.builder("FREQ", "AME_REF_AREA", "AME_TRANSFORMATION", "AME_AGG_METHOD", "AME_UNIT", "AME_REFERENCE", "AME_ITEM"); - DataCursorAssert.assertCompliance(() -> XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))); + DataCursorAssert.assertCompliance(() -> { + InputStream stream = xml.openStream(); + return XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + }); - try (DataCursor o = XMLStreamGenericDataCursor.sdmx21(xml.openXmlStream(SdmxSource.XIF), builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + try (InputStream stream = xml.openStream(); + DataCursor o = XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index 15e9d5db..e77c4851 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -22,7 +22,6 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import java.util.List; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -37,11 +36,9 @@ public class XMLStreamStructure20Test { @Test @SuppressWarnings("null") public void test() throws Exception { - XMLInputFactory factory = XMLInputFactory.newFactory(); - XMLStream> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - assertThat(p1.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(p1.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -55,7 +52,7 @@ public void test() throws Exception { XMLStream> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - assertThat(p2.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(p2.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -67,7 +64,7 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> p1.get(factory, SdmxSource.ECB_DATA_STRUCTURE.openReader())) + assertThatThrownBy(() -> p1.parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader)) .isInstanceOf(IOException.class) .hasCauseInstanceOf(XMLStreamException.class) .hasMessageContaining("Invalid namespace"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index 438f1251..6fee84b8 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -22,7 +22,6 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import java.util.List; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -37,11 +36,9 @@ public class XMLStreamStructure21Test { @Test @SuppressWarnings("null") public void test() throws Exception { - XMLInputFactory factory = XMLInputFactory.newFactory(); - XMLStream> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - assertThat(parser.get(factory, SdmxSource.ECB_DATA_STRUCTURE.openReader())).hasSize(1).element(0).satisfies(o -> { + assertThat(parser.parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); @@ -53,7 +50,7 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> parser.get(factory, SdmxSource.NBB_DATA_STRUCTURE.openReader())) + assertThatThrownBy(() -> parser.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)) .isInstanceOf(IOException.class) .hasCauseInstanceOf(XMLStreamException.class) .hasMessageContaining("Invalid namespace"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java new file mode 100644 index 00000000..bcea3857 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java @@ -0,0 +1,150 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import be.nbb.sdmx.facade.samples.SdmxSource; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import static java.nio.charset.StandardCharsets.UTF_8; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class XMLStreamTest { + + @Test + @SuppressWarnings("null") + public void testValidParseStream() throws IOException { + XMLStream p = (o, c) -> { + c.close(); + return ""; + }; + + assertThatThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)) + .isInstanceOf(NullPointerException.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, null, UTF_8)) + .isInstanceOf(NullPointerException.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) + .isInstanceOf(OpenError.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) + .isInstanceOf(IOException.class) + .hasNoSuppressedExceptions() + .hasCauseInstanceOf(XMLStreamException.class) + .hasRootCauseInstanceOf(ReadError.class); + + assertThatThrownBy(() -> p.parseStream(xif, CloseErrorStream::new, UTF_8)) + .isInstanceOf(CloseError.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + } + + @Test + @SuppressWarnings("null") + public void testInvalidParseStream() throws IOException { + XMLStream p = (o, c) -> { + try (Closeable x = c) { + throw new ParseError(); + } + }; + + assertThatThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)) + .isInstanceOf(NullPointerException.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, null, UTF_8)) + .isInstanceOf(NullPointerException.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) + .isInstanceOf(OpenError.class) + .hasNoSuppressedExceptions() + .hasNoCause(); + + assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) + .isInstanceOf(IOException.class) + .hasNoSuppressedExceptions() + .hasCauseInstanceOf(XMLStreamException.class) + .hasRootCauseInstanceOf(ReadError.class); + + assertThatThrownBy(() -> p.parseStream(xif, CloseErrorStream::new, UTF_8)) + .isInstanceOf(ParseError.class) + .hasSuppressedException(new CloseError()) + .hasNoCause(); + } + + private final XMLInputFactory xif = SdmxSource.XIF; + + private static class EmptyStream extends InputStream { + + @Override + public int read() throws IOException { + return -1; + } + } + + private static final class OpenErrorStream extends EmptyStream { + + public OpenErrorStream() throws IOException { + throw new OpenError(); + } + } + + private static final class ReadErrorStream extends EmptyStream { + + @Override + public int read() throws IOException { + throw new ReadError(); + } + } + + private static final class CloseErrorStream extends EmptyStream { + + @Override + public void close() throws IOException { + throw new CloseError(); + } + } + + private static final class ParseError extends IOException { + } + + private static final class OpenError extends IOException { + } + + private static final class ReadError extends IOException { + } + + private static final class CloseError extends IOException { + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index ddbd2b44..8bf51a87 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -114,7 +114,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd), dataFactory).get(new XMLEventStreamReader(r))) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd), dataFactory).parse(new XMLEventStreamReader(r), () -> {})) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 481b0206..c20d54ef 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -205,7 +205,7 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).get(new XMLEventStreamReader(r))) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).parse(new XMLEventStreamReader(r), () -> {})) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index 1ce6d815..51b0189e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -39,7 +39,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; /** @@ -103,7 +102,7 @@ List flow20(ByteSource xml, LanguagePriorityList l) throws IOException List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data20(XMLInputFactory.newFactory(), xml, Util.toStructure(dsd)) + return FacadeResource.data20(SdmxSource.XIF, xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -119,7 +118,7 @@ List flow21(ByteSource xml, LanguagePriorityList l) throws IOException List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data21(XMLInputFactory.newFactory(), xml, Util.toStructure(dsd)) + return FacadeResource.data21(SdmxSource.XIF, xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -168,7 +167,7 @@ Dataflow asDataflow(DataFlowStructure o) { T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { XMLEventReader r = null; try { - r = xml.openXmlEvent(SdmxSource.XIF); + r = SdmxSource.XIF.createXMLEventReader(xml.openReader()); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index a083d287..f36fbb06 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -28,7 +28,6 @@ import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; -import java.io.InputStreamReader; import java.util.List; import java.util.stream.Collectors; import javax.xml.stream.XMLInputFactory; @@ -41,7 +40,7 @@ public class FacadeResource { public SdmxRepository nbb() throws IOException { - XMLInputFactory f = XMLInputFactory.newFactory(); + XMLInputFactory f = SdmxSource.XIF; LanguagePriorityList l = LanguagePriorityList.parse("fr"); List structs = struct20(f, SdmxSource.NBB_DATA_STRUCTURE, l); @@ -60,7 +59,7 @@ public SdmxRepository nbb() throws IOException { } public SdmxRepository ecb() throws IOException { - XMLInputFactory f = XMLInputFactory.newFactory(); + XMLInputFactory f = SdmxSource.XIF; LanguagePriorityList l = LanguagePriorityList.parse("fr"); List structs = struct21(f, SdmxSource.ECB_DATA_STRUCTURE, l); @@ -79,9 +78,7 @@ public SdmxRepository ecb() throws IOException { } private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return SdmxXmlStreams.struct20(l).get(f, r); - } + return SdmxXmlStreams.struct20(l).parseReader(f, xml::openReader); } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -91,15 +88,13 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).get(f, xml.openReader())) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(f, xml::openReader)) { return SeriesSupport.copyOf(c); } } private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - try (InputStreamReader r = xml.openReader()) { - return SdmxXmlStreams.struct21(l).get(f, r); - } + return SdmxXmlStreams.struct21(l).parseReader(f, xml::openReader); } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -111,7 +106,7 @@ private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).get(f, xml.openReader())) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(f, xml::openReader)) { return SeriesSupport.copyOf(c); } } From 96f03ab6af71aaf63fdc4c807dbb99db6d25a5e7 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 15 Nov 2017 16:10:37 +0100 Subject: [PATCH 36/68] Improved xml parsing speed by letting Stax handling input buffers. --- .../internal/file/DataStructureDecoder.java | 27 +++++++--------- .../internal/file/FileSdmxConnection.java | 2 +- .../internal/file/XMLStreamSdmxDecoder.java | 32 +++++++++++++------ .../file/DataStructureDecoderTest.java | 20 +++--------- .../nbb/sdmx/facade/xml/stream/XMLStream.java | 13 +++++++- 5 files changed, 51 insertions(+), 43 deletions(-) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index 63cb236d..86465153 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -31,23 +31,20 @@ */ final class DataStructureDecoder { - public static XMLStream of(SdmxDecoder.DataType dataType) { - return XMLStream.of(o -> decodeDataStructure(dataType, o)); + public static XMLStream generic20() { + return XMLStream.of(DataStructureDecoder::generic20); } - private static DataStructure decodeDataStructure(SdmxDecoder.DataType dataType, XMLStreamReader reader) throws XMLStreamException { - switch (dataType) { - case GENERIC20: - return generic20(reader); - case COMPACT20: - return compact20(reader); - case GENERIC21: - return generic21(reader); - case COMPACT21: - return compact21(reader); - default: - throw new XMLStreamException("Don't know how to handle '" + dataType + "'"); - } + public static XMLStream compact20() { + return XMLStream.of(DataStructureDecoder::compact20); + } + + public static XMLStream generic21() { + return XMLStream.of(DataStructureDecoder::generic21); + } + + public static XMLStream compact21() { + return XMLStream.of(DataStructureDecoder::compact21); } // diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index a1aaa596..b965c180 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -120,7 +120,7 @@ protected SdmxDecoder.Info decode() throws IOException { } protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factory, files.getData().toPath(), StandardCharsets.UTF_8); + return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factory, files.getData(), StandardCharsets.UTF_8); } private XMLStream getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index 196a149d..939ca97e 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -35,31 +35,28 @@ * * @author Philippe Charles */ +@lombok.AllArgsConstructor public final class XMLStreamSdmxDecoder implements SdmxDecoder { private final XMLInputFactory factory; - public XMLStreamSdmxDecoder(XMLInputFactory factory) { - this.factory = factory; - } - @Override public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { DataType dataType = probeDataType(files.getData()); return Info.of(dataType, files.getStructure() != null - ? parseStruct(dataType, files.getStructure(), langs) + ? parseStruct(dataType, langs, files.getStructure()) : decodeStruct(dataType, files.getData())); } private DataType probeDataType(File data) throws IOException { - return DataTypeProbe.of().parseFile(factory, data.toPath(), StandardCharsets.UTF_8); + return DataTypeProbe.of().parseFile(factory, data, StandardCharsets.UTF_8); } - private DataStructure parseStruct(DataType dataType, File structure, LanguagePriorityList langs) throws IOException { - return getStructSupplier(dataType, langs).parseFile(factory, structure.toPath(), StandardCharsets.UTF_8).get(0); + private DataStructure parseStruct(DataType dataType, LanguagePriorityList langs, File structure) throws IOException { + return getStructParser(dataType, langs).parseFile(factory, structure, StandardCharsets.UTF_8).get(0); } - private XMLStream> getStructSupplier(DataType o, LanguagePriorityList langs) throws IOException { + private XMLStream> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { switch (o) { case GENERIC20: case COMPACT20: @@ -73,6 +70,21 @@ private XMLStream> getStructSupplier(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - return DataStructureDecoder.of(dataType).parseFile(factory, data.toPath(), StandardCharsets.UTF_8); + return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); + } + + private static XMLStream getStructDecoder(SdmxDecoder.DataType o) throws IOException { + switch (o) { + case GENERIC20: + return DataStructureDecoder.generic20(); + case COMPACT20: + return DataStructureDecoder.compact20(); + case GENERIC21: + return DataStructureDecoder.generic21(); + case COMPACT21: + return DataStructureDecoder.compact21(); + default: + throw new IOException("Don't know how to handle '" + o + "'"); + } } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index bd8a6a37..71820c2d 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -43,10 +43,7 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder - .of(SdmxDecoder.DataType.GENERIC20) - .parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader) - ).isEqualTo(ds); + assertThat(DataStructureDecoder.generic20().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); } @Test @@ -63,10 +60,7 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder - .of(SdmxDecoder.DataType.COMPACT20) - .parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader) - ).isEqualTo(ds); + assertThat(DataStructureDecoder.compact20().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); } @Test @@ -85,10 +79,7 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder - .of(SdmxDecoder.DataType.GENERIC21) - .parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader) - ).isEqualTo(ds); + assertThat(DataStructureDecoder.generic21().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); } @Test @@ -107,9 +98,6 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder - .of(SdmxDecoder.DataType.COMPACT21) - .parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader) - ).isEqualTo(ds); + assertThat(DataStructureDecoder.compact21().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java index 185f422d..b63283c6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java @@ -18,6 +18,8 @@ import be.nbb.sdmx.facade.util.IO; import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -36,9 +38,18 @@ */ public interface XMLStream { + @Nonnull + default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull File file, @Nonnull Charset cs) throws IOException { + return parseStream(xf, () -> new FileInputStream(file), cs); + } + @Nonnull default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { - return parseReader(xf, () -> Files.newBufferedReader(path, cs)); + try { + return parseStream(xf, () -> new FileInputStream(path.toFile()), cs); + } catch (UnsupportedOperationException ex) { + return parseReader(xf, () -> Files.newBufferedReader(path, cs)); + } } @Nonnull From d5d393daf1268d102ddab640c99f587fa7148125 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 10:59:40 +0100 Subject: [PATCH 37/68] Improved xml parsing speed by avoiding Stax namespace parsing. --- .../internal/file/DataStructureDecoder.java | 177 ++++++++---------- .../internal/file/XMLStreamSdmxDecoder.java | 12 +- 2 files changed, 85 insertions(+), 104 deletions(-) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index 86465153..db9f70d4 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -47,16 +47,18 @@ public static XMLStream compact21() { return XMLStream.of(DataStructureDecoder::compact21); } + private static boolean isTagMatch(XMLStreamReader r, String tag) { + return r.getLocalName().endsWith(tag); + } + // private static DataStructure generic20(XMLStreamReader reader) throws XMLStreamException { CustomDataStructureBuilder builder = new CustomDataStructureBuilder().fileType(GENERIC20); while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "DataSet": - generic20DataSet(reader, builder); - break; + if (isTagMatch(reader, "DataSet")) { + generic20DataSet(reader, builder); } break; } @@ -68,17 +70,14 @@ private static void generic20DataSet(XMLStreamReader reader, CustomDataStructure while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "KeyFamilyRef": - builder.refId(reader.getElementText()); - break; - case "Series": - generic20Series(reader, builder); - break; + if (isTagMatch(reader, "KeyFamilyRef")) { + builder.refId(reader.getElementText()); + } else if (isTagMatch(reader, "Series")) { + generic20Series(reader, builder); } break; case END_ELEMENT: - if (reader.getLocalName().equals("DataSet")) { + if (isTagMatch(reader, "DataSet")) { return; } break; @@ -90,17 +89,14 @@ private static void generic20Series(XMLStreamReader reader, CustomDataStructureB while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "SeriesKey": - generic20SeriesKey(reader, builder); - break; - case "Attributes": - generic20Attributes(reader, builder); - break; + if (isTagMatch(reader, "SeriesKey")) { + generic20SeriesKey(reader, builder); + } else if (isTagMatch(reader, "Attributes")) { + generic20Attributes(reader, builder); } break; case END_ELEMENT: - if (reader.getLocalName().equals("Series")) { + if (isTagMatch(reader, "Series")) { return; } break; @@ -112,14 +108,12 @@ private static void generic20SeriesKey(XMLStreamReader reader, CustomDataStructu while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Value": - builder.dimension(reader.getAttributeValue(null, "concept"), reader.getAttributeValue(null, "value")); - break; + if (isTagMatch(reader, "Value")) { + builder.dimension(reader.getAttributeValue(null, "concept"), reader.getAttributeValue(null, "value")); } break; case END_ELEMENT: - if (reader.getLocalName().equals("SeriesKey")) { + if (isTagMatch(reader, "SeriesKey")) { return; } break; @@ -131,14 +125,12 @@ private static void generic20Attributes(XMLStreamReader reader, CustomDataStruct while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Value": - builder.attribute(reader.getAttributeValue(null, "concept"), reader.getAttributeValue(null, "value")); - break; + if (isTagMatch(reader, "Value")) { + builder.attribute(reader.getAttributeValue(null, "concept"), reader.getAttributeValue(null, "value")); } break; case END_ELEMENT: - if (reader.getLocalName().equals("Attributes")) { + if (isTagMatch(reader, "Attributes")) { return; } break; @@ -153,13 +145,10 @@ private static DataStructure compact20(XMLStreamReader reader) throws XMLStreamE while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "CompactData": - builder.refId("TODO"); - break; - case "DataSet": - compact20DataSet(reader, builder); - break; + if (isTagMatch(reader, "CompactData")) { + builder.refId("TODO"); + } else if (isTagMatch(reader, "DataSet")) { + compact20DataSet(reader, builder); } break; } @@ -171,21 +160,19 @@ private static void compact20DataSet(XMLStreamReader reader, CustomDataStructure while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Series": - for (int i = 0; i < reader.getAttributeCount(); i++) { - String concept = reader.getAttributeLocalName(i); - if (concept.equals(TIME_FORMAT_CONCEPT)) { - builder.attribute(concept, reader.getAttributeValue(i)); - } else { - builder.dimension(concept, reader.getAttributeValue(i)); - } + if (isTagMatch(reader, "Series")) { + for (int i = 0; i < reader.getAttributeCount(); i++) { + String concept = reader.getAttributeLocalName(i); + if (concept.equals(TIME_FORMAT_CONCEPT)) { + builder.attribute(concept, reader.getAttributeValue(i)); + } else { + builder.dimension(concept, reader.getAttributeValue(i)); } - break; + } } break; case END_ELEMENT: - if (reader.getLocalName().equals("DataSet")) { + if (isTagMatch(reader, "DataSet")) { return; } break; @@ -200,10 +187,8 @@ private static DataStructure generic21(XMLStreamReader reader) throws XMLStreamE while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "DataSet": - generic21DataSet(reader, builder); - break; + if (isTagMatch(reader, "DataSet")) { + generic21DataSet(reader, builder); } break; } @@ -216,14 +201,12 @@ private static void generic21DataSet(XMLStreamReader reader, CustomDataStructure while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Series": - generic21Series(reader, builder); - break; + if (isTagMatch(reader, "Series")) { + generic21Series(reader, builder); } break; case END_ELEMENT: - if (reader.getLocalName().equals("DataSet")) { + if (isTagMatch(reader, "DataSet")) { return; } break; @@ -235,17 +218,14 @@ private static void generic21Series(XMLStreamReader reader, CustomDataStructureB while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "SeriesKey": - generic21SeriesKey(reader, builder); - break; - case "Attributes": - generic21Attributes(reader, builder); - break; + if (isTagMatch(reader, "SeriesKey")) { + generic21SeriesKey(reader, builder); + } else if (isTagMatch(reader, "Attributes")) { + generic21Attributes(reader, builder); } break; case END_ELEMENT: - if (reader.getLocalName().equals("Series")) { + if (isTagMatch(reader, "Series")) { return; } break; @@ -257,14 +237,12 @@ private static void generic21SeriesKey(XMLStreamReader reader, CustomDataStructu while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Value": - builder.dimension(reader.getAttributeValue(null, "id"), reader.getAttributeValue(null, "value")); - break; + if (isTagMatch(reader, "Value")) { + builder.dimension(reader.getAttributeValue(null, "id"), reader.getAttributeValue(null, "value")); } break; case END_ELEMENT: - if (reader.getLocalName().equals("SeriesKey")) { + if (isTagMatch(reader, "SeriesKey")) { return; } break; @@ -276,14 +254,12 @@ private static void generic21Attributes(XMLStreamReader reader, CustomDataStruct while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Value": - builder.attribute(reader.getAttributeValue(null, "id"), reader.getAttributeValue(null, "value")); - break; + if (isTagMatch(reader, "Value")) { + builder.attribute(reader.getAttributeValue(null, "id"), reader.getAttributeValue(null, "value")); } break; case END_ELEMENT: - if (reader.getLocalName().equals("Attributes")) { + if (isTagMatch(reader, "Attributes")) { return; } break; @@ -298,13 +274,10 @@ private static DataStructure compact21(XMLStreamReader reader) throws XMLStreamE while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Header": - compact21Header(reader, builder); - break; - case "DataSet": - compact21DataSet(reader, builder); - break; + if (isTagMatch(reader, "Header")) { + compact21Header(reader, builder); + } else if (isTagMatch(reader, "DataSet")) { + compact21DataSet(reader, builder); } break; } @@ -316,19 +289,18 @@ private static void compact21Header(XMLStreamReader reader, CustomDataStructureB while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Structure": - String structureId = reader.getAttributeValue(null, "structureID"); - String dimensionAtObservation = reader.getAttributeValue(null, "dimensionAtObservation"); - if (structureId != null && dimensionAtObservation != null) { - builder.refId(structureId); - builder.timeDimensionId(dimensionAtObservation); - } - break; + if (isTagMatch(reader, "Structure")) { + String structureId = reader.getAttributeValue(null, "structureID"); + String dimensionAtObservation = reader.getAttributeValue(null, "dimensionAtObservation"); + if (structureId != null && dimensionAtObservation != null) { + builder.refId(structureId); + builder.timeDimensionId(dimensionAtObservation); + } + } break; case END_ELEMENT: - if (reader.getLocalName().equals("Header")) { + if (isTagMatch(reader, "Header")) { return; } break; @@ -340,21 +312,20 @@ private static void compact21DataSet(XMLStreamReader reader, CustomDataStructure while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - switch (reader.getLocalName()) { - case "Series": - for (int i = 0; i < reader.getAttributeCount(); i++) { - String concept = reader.getAttributeLocalName(i); - if (concept.equals(TIME_FORMAT_CONCEPT)) { - builder.attribute(concept, reader.getAttributeValue(i)); - } else { - builder.dimension(concept, reader.getAttributeValue(i)); - } + if (isTagMatch(reader, "Series")) { + for (int i = 0; i < reader.getAttributeCount(); i++) { + String concept = reader.getAttributeLocalName(i); + if (concept.equals(TIME_FORMAT_CONCEPT)) { + builder.attribute(concept, reader.getAttributeValue(i)); + } else { + builder.dimension(concept, reader.getAttributeValue(i)); } - break; + } + } break; case END_ELEMENT: - if (reader.getLocalName().equals("DataSet")) { + if (isTagMatch(reader, "DataSet")) { return; } break; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index 939ca97e..3cadbe13 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -70,7 +70,17 @@ private XMLStream> getStructParser(DataType o, LanguagePrior } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); + if (factory.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE) + && (Boolean) factory.getProperty(XMLInputFactory.IS_NAMESPACE_AWARE)) { + factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); + try { + return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); + } finally { + factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true); + } + } else { + return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); + } } private static XMLStream getStructDecoder(SdmxDecoder.DataType o) throws IOException { From aa16a1fff3d5b9937107cba8584e6bd19e4ee9e8 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 11:55:34 +0100 Subject: [PATCH 38/68] Refactored Stax api. --- .../internal/file/DataStructureDecoder.java | 18 +-- .../java/internal/file/DataTypeProbe.java | 6 +- .../internal/file/FileSdmxConnection.java | 4 +- .../internal/file/XMLStreamSdmxDecoder.java | 6 +- .../file/DataStructureDecoderTest.java | 1 + .../facade/xml/stream/SdmxXmlStreams.java | 24 ++-- .../be/nbb/sdmx/facade/xml/stream/Stax.java | 136 ++++++++++++++++++ .../be/nbb/sdmx/facade/xml/stream/XFunc.java | 30 ---- .../nbb/sdmx/facade/xml/stream/XMLStream.java | 94 ------------ .../stream/XMLStreamCompactDataCursor.java | 6 +- .../stream/XMLStreamGenericDataCursor.java | 6 +- .../xml/stream/XMLStreamStructure20.java | 2 + .../xml/stream/XMLStreamStructure21.java | 2 + .../sdmx/facade/xml/stream/XMLStreamUtil.java | 20 --- .../{XMLStreamTest.java => StaxTest.java} | 10 +- .../xml/stream/XMLStreamStructure20Test.java | 4 +- .../xml/stream/XMLStreamStructure21Test.java | 2 +- 17 files changed, 184 insertions(+), 187 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/{XMLStreamTest.java => StaxTest.java} (95%) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index db9f70d4..3fca3d61 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -19,11 +19,11 @@ import be.nbb.sdmx.facade.DataStructure; import static internal.file.SdmxDecoder.DataType.*; import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; -import be.nbb.sdmx.facade.xml.stream.XMLStream; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import be.nbb.sdmx.facade.xml.stream.Stax; /** * @@ -31,20 +31,20 @@ */ final class DataStructureDecoder { - public static XMLStream generic20() { - return XMLStream.of(DataStructureDecoder::generic20); + public static Stax.Parser generic20() { + return Stax.Parser.of(DataStructureDecoder::generic20); } - public static XMLStream compact20() { - return XMLStream.of(DataStructureDecoder::compact20); + public static Stax.Parser compact20() { + return Stax.Parser.of(DataStructureDecoder::compact20); } - public static XMLStream generic21() { - return XMLStream.of(DataStructureDecoder::generic21); + public static Stax.Parser generic21() { + return Stax.Parser.of(DataStructureDecoder::generic21); } - public static XMLStream compact21() { - return XMLStream.of(DataStructureDecoder::compact21); + public static Stax.Parser compact21() { + return Stax.Parser.of(DataStructureDecoder::compact21); } private static boolean isTagMatch(XMLStreamReader r, String tag) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java index da47ee2a..be1a3894 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java @@ -17,12 +17,12 @@ package internal.file; import static be.nbb.sdmx.facade.xml.Sdmxml.*; -import be.nbb.sdmx.facade.xml.stream.XMLStream; import static internal.file.SdmxDecoder.DataType.*; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import be.nbb.sdmx.facade.xml.stream.Stax; /** * @@ -30,8 +30,8 @@ */ final class DataTypeProbe { - public static XMLStream of() { - return XMLStream.of(DataTypeProbe::probeDataType); + public static Stax.Parser of() { + return Stax.Parser.of(DataTypeProbe::probeDataType); } private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index b965c180..6b895fbd 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -30,7 +30,6 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -38,6 +37,7 @@ import java.util.Set; import java.util.stream.Stream; import javax.xml.stream.XMLInputFactory; +import be.nbb.sdmx.facade.xml.stream.Stax; /** * @@ -123,7 +123,7 @@ protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key k return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factory, files.getData(), StandardCharsets.UTF_8); } - private XMLStream getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { + private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { switch (o) { case GENERIC20: return SdmxXmlStreams.genericData20(dsd); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index 3cadbe13..63e68751 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -28,8 +28,8 @@ import static internal.file.SdmxDecoder.DataType.GENERIC21; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.XMLStream; import java.util.List; +import be.nbb.sdmx.facade.xml.stream.Stax; /** * @@ -56,7 +56,7 @@ private DataStructure parseStruct(DataType dataType, LanguagePriorityList langs, return getStructParser(dataType, langs).parseFile(factory, structure, StandardCharsets.UTF_8).get(0); } - private XMLStream> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { + private Stax.Parser> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { switch (o) { case GENERIC20: case COMPACT20: @@ -83,7 +83,7 @@ private DataStructure decodeStruct(DataType dataType, File data) throws IOExcept } } - private static XMLStream getStructDecoder(SdmxDecoder.DataType o) throws IOException { + private static Stax.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { switch (o) { case GENERIC20: return DataStructureDecoder.generic20(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index 71820c2d..16ad9850 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import static org.assertj.core.api.Assertions.assertThat; import static internal.file.CustomDataStructureBuilder.dimension; +import static internal.file.CustomDataStructureBuilder.dimension; /** * diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index d361a808..2350e20e 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -33,52 +33,52 @@ public class SdmxXmlStreams { @Nonnull - public XMLStream compactData20(@Nonnull DataStructure dsd) throws IOException { + public Stax.Parser compactData20(@Nonnull DataStructure dsd) throws IOException { return compactData20(dsd, DataFactory.sdmx20()); } @Nonnull - public XMLStream compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Stax.Parser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); } @Nonnull - public XMLStream compactData21(@Nonnull DataStructure dsd) throws IOException { + public Stax.Parser compactData21(@Nonnull DataStructure dsd) throws IOException { return compactData21(dsd, DataFactory.sdmx21()); } @Nonnull - public XMLStream compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Stax.Parser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull - public XMLStream genericData20(@Nonnull DataStructure dsd) throws IOException { + public Stax.Parser genericData20(@Nonnull DataStructure dsd) throws IOException { return genericData20(dsd, DataFactory.sdmx20()); } @Nonnull - public XMLStream genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Stax.Parser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return (o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull - public XMLStream genericData21(@Nonnull DataStructure dsd) throws IOException { + public Stax.Parser genericData21(@Nonnull DataStructure dsd) throws IOException { return genericData21(dsd, DataFactory.sdmx21()); } @Nonnull - public XMLStream genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Stax.Parser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return (o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); } @Nonnull - public XMLStream> struct20(@Nonnull LanguagePriorityList langs) throws IOException { - return XMLStream.of(new XMLStreamStructure20(langs)::parse); + public Stax.Parser> struct20(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.Parser.of(new XMLStreamStructure20(langs)::parse); } @Nonnull - public XMLStream> struct21(@Nonnull LanguagePriorityList langs) throws IOException { - return XMLStream.of(new XMLStreamStructure21(langs)::parse); + public Stax.Parser> struct21(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.Parser.of(new XMLStreamStructure21(langs)::parse); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java new file mode 100644 index 00000000..9637ae4b --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java @@ -0,0 +1,136 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import be.nbb.sdmx.facade.util.IO; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.annotation.Nonnull; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class Stax { + + @FunctionalInterface + public interface Supplier { + + T getWithStream() throws XMLStreamException; + } + + @FunctionalInterface + public interface Function { + + R applyWithStream(T t) throws XMLStreamException; + } + + @FunctionalInterface + public interface Parser { + + @Nonnull + default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull File file, @Nonnull Charset cs) throws IOException { + return parseStream(xf, () -> new FileInputStream(file), cs); + } + + @Nonnull + default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { + try { + return parseStream(xf, () -> new FileInputStream(path.toFile()), cs); + } catch (UnsupportedOperationException ex) { + return parseReader(xf, () -> Files.newBufferedReader(path, cs)); + } + } + + @Nonnull + default T parseStream(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source, @Nonnull Charset cs) throws IOException { + InputStream stream = source.getWithIO(); + return parse(() -> xf.createXMLStreamReader(stream, cs.name()), stream); + } + + @Nonnull + default T parseReader(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source) throws IOException { + Reader reader = source.getWithIO(); + return parse(() -> xf.createXMLStreamReader(reader), reader); + } + + @Nonnull + default T parse(@Nonnull Supplier supplier, @Nonnull Closeable onClose) throws IOException { + try { + XMLStreamReader xml = supplier.getWithStream(); + return parse(xml, () -> closeBoth(xml, onClose)); + } catch (XMLStreamException ex) { + ensureClosed(ex, onClose); + throw new XMLStreamIOException("Failed to create XMLStreamReader", ex); + } + } + + @Nonnull + T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; + + @Nonnull + static Parser of(@Nonnull Function func) { + return (reader, onClose) -> { + try (Closeable c = onClose) { + return func.applyWithStream(reader); + } catch (XMLStreamException ex) { + throw new XMLStreamIOException(ex); + } + }; + } + } + + public static final class XMLStreamIOException extends IOException { + + public XMLStreamIOException(XMLStreamException ex) { + super(ex); + } + + public XMLStreamIOException(String message, XMLStreamException ex) { + super(message, ex); + } + } + + void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { + try { + reader.close(); + } catch (XMLStreamException ex) { + ensureClosed(ex, onClose); + throw new Stax.XMLStreamIOException("Failed to close xml stream reader", ex); + } + onClose.close(); + } + + void ensureClosed(XMLStreamException ex, Closeable onClose) throws IOException { + try { + onClose.close(); + } catch (IOException suppressed) { + ex.addSuppressed(suppressed); + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java deleted file mode 100644 index 4c14cad7..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XFunc.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.xml.stream; - -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; - -/** - * - * @author Philippe Charles - * @param - */ -public interface XFunc { - - T apply(XMLStreamReader reader) throws XMLStreamException; -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java deleted file mode 100644 index b63283c6..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStream.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.xml.stream; - -import be.nbb.sdmx.facade.util.IO; -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import javax.annotation.Nonnull; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; - -/** - * - * @author Philippe Charles - * @param - */ -public interface XMLStream { - - @Nonnull - default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull File file, @Nonnull Charset cs) throws IOException { - return parseStream(xf, () -> new FileInputStream(file), cs); - } - - @Nonnull - default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { - try { - return parseStream(xf, () -> new FileInputStream(path.toFile()), cs); - } catch (UnsupportedOperationException ex) { - return parseReader(xf, () -> Files.newBufferedReader(path, cs)); - } - } - - @Nonnull - default T parseStream(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier supplier, @Nonnull Charset cs) throws IOException { - InputStream stream = supplier.getWithIO(); - - try { - XMLStreamReader xml = xf.createXMLStreamReader(stream, cs.name()); - return parse(xml, () -> XMLStreamUtil.closeBoth(xml, stream)); - } catch (XMLStreamException ex) { - XMLStreamUtil.ensureClosed(ex, stream); - throw new IOException("Failed to create XMLStreamReader from a stream", ex); - } - } - - @Nonnull - default T parseReader(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier supplier) throws IOException { - Reader reader = supplier.getWithIO(); - - try { - XMLStreamReader xml = xf.createXMLStreamReader(reader); - return parse(xml, () -> XMLStreamUtil.closeBoth(xml, reader)); - } catch (XMLStreamException ex) { - XMLStreamUtil.ensureClosed(ex, reader); - throw new IOException("Failed to create XMLStreamReader from a reader", ex); - } - } - - @Nonnull - T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; - - @Nonnull - static XMLStream of(@Nonnull XFunc func) { - return (reader, onClose) -> { - try (Closeable c = onClose) { - return func.apply(reader); - } catch (XMLStreamException ex) { - throw new IOException(ex); - } - }; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 2ce700a8..c0f94230 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -76,7 +76,7 @@ public boolean nextSeries() throws IOException { try { return hasSeries = nextWhile(this::onDataSet); } catch (XMLStreamException ex) { - throw new IOException(ex); + throw new Stax.XMLStreamIOException(ex); } } @@ -87,7 +87,7 @@ public boolean nextObs() throws IOException { try { return hasObs = nextWhile(this::onSeriesBody); } catch (XMLStreamException ex) { - throw new IOException(ex); + throw new Stax.XMLStreamIOException(ex); } } @@ -133,7 +133,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - XMLStreamUtil.closeBoth(reader, onClose); + Stax.closeBoth(reader, onClose); } private void checkState() throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 1ff5eea2..409164f4 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -90,7 +90,7 @@ public boolean nextSeries() throws IOException { try { return hasSeries = nextWhile(this::onDataSet); } catch (XMLStreamException ex) { - throw new IOException(ex); + throw new Stax.XMLStreamIOException(ex); } } @@ -108,7 +108,7 @@ public boolean nextObs() throws IOException { } return hasObs = nextWhile(this::onSeriesBody); } catch (XMLStreamException ex) { - throw new IOException(ex); + throw new Stax.XMLStreamIOException(ex); } } @@ -154,7 +154,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - XMLStreamUtil.closeBoth(reader, onClose); + Stax.closeBoth(reader, onClose); } private void checkState() throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index 4715ee48..1b0318dd 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -32,11 +32,13 @@ import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V20_URI; import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; /** * * @author Philippe Charles */ +@NotThreadSafe final class XMLStreamStructure20 { private static final String HEADER_TAG = "Header"; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 97f4fed1..c325e1da 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -32,11 +32,13 @@ import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; /** * * @author Philippe Charles */ +@NotThreadSafe final class XMLStreamStructure21 { private static final String HEADER_TAG = "Header"; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java index 504e82f6..45377d03 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java @@ -16,8 +16,6 @@ */ package be.nbb.sdmx.facade.xml.stream; -import java.io.Closeable; -import java.io.IOException; import java.util.function.Supplier; import javax.xml.stream.Location; import javax.xml.stream.XMLStreamException; @@ -107,22 +105,4 @@ static void check(boolean expression, Supplier location, String messag throw new XMLStreamException(String.format(message, args), location.get()); } } - - void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { - try { - reader.close(); - } catch (XMLStreamException ex) { - ensureClosed(ex, onClose); - throw new IOException("Failed to close xml stream reader", ex); - } - onClose.close(); - } - - void ensureClosed(XMLStreamException ex, Closeable onClose) throws IOException { - try { - onClose.close(); - } catch (IOException suppressed) { - ex.addSuppressed(suppressed); - } - } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java similarity index 95% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java index bcea3857..0faba5a6 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java @@ -30,12 +30,12 @@ * * @author Philippe Charles */ -public class XMLStreamTest { +public class StaxTest { @Test @SuppressWarnings("null") public void testValidParseStream() throws IOException { - XMLStream p = (o, c) -> { + Stax.Parser p = (o, c) -> { c.close(); return ""; }; @@ -56,7 +56,7 @@ public void testValidParseStream() throws IOException { .hasNoCause(); assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) - .isInstanceOf(IOException.class) + .isInstanceOf(Stax.XMLStreamIOException.class) .hasNoSuppressedExceptions() .hasCauseInstanceOf(XMLStreamException.class) .hasRootCauseInstanceOf(ReadError.class); @@ -70,7 +70,7 @@ public void testValidParseStream() throws IOException { @Test @SuppressWarnings("null") public void testInvalidParseStream() throws IOException { - XMLStream p = (o, c) -> { + Stax.Parser p = (o, c) -> { try (Closeable x = c) { throw new ParseError(); } @@ -92,7 +92,7 @@ public void testInvalidParseStream() throws IOException { .hasNoCause(); assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) - .isInstanceOf(IOException.class) + .isInstanceOf(Stax.XMLStreamIOException.class) .hasNoSuppressedExceptions() .hasCauseInstanceOf(XMLStreamException.class) .hasRootCauseInstanceOf(ReadError.class); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index e77c4851..4b2a1dd3 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -36,7 +36,7 @@ public class XMLStreamStructure20Test { @Test @SuppressWarnings("null") public void test() throws Exception { - XMLStream> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); + Stax.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); assertThat(p1.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); @@ -50,7 +50,7 @@ public void test() throws Exception { }); }); - XMLStream> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); + Stax.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); assertThat(p2.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index 6fee84b8..76b4ced7 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -36,7 +36,7 @@ public class XMLStreamStructure21Test { @Test @SuppressWarnings("null") public void test() throws Exception { - XMLStream> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); + Stax.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); assertThat(parser.parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); From 638907d45c08bc627cc047a67038132bda87ce5d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 14:03:54 +0100 Subject: [PATCH 39/68] Fixed XMLInputFactory threading issue. --- .../internal/connectors/TestResource.java | 10 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 5 +- .../java/internal/file/DataTypeProbe.java | 4 + .../main/java/internal/file/SdmxFileUtil.java | 5 +- .../internal/file/XMLStreamSdmxDecoder.java | 13 +- .../file/DataStructureDecoderTest.java | 13 +- .../java/internal/file/DataTypeProbeTest.java | 12 +- .../internal/file/FileSdmxConnectionTest.java | 16 +- .../nbb/sdmx/facade/samples/SdmxSource.java | 4 - .../be/nbb/sdmx/facade/xml/stream/Stax.java | 156 ++++++++++++++++++ .../xml/stream/XMLStreamStructure20.java | 4 + .../xml/stream/XMLStreamStructure21.java | 4 + .../sdmx/facade/util/SeriesSupportTest.java | 11 +- .../nbb/sdmx/facade/xml/stream/StaxTest.java | 2 +- .../XMLStreamCompactDataCursorTest.java | 11 +- .../XMLStreamGenericDataCursorTest.java | 15 +- .../xml/stream/XMLStreamStructure20Test.java | 9 +- .../xml/stream/XMLStreamStructure21Test.java | 7 +- .../connectors/ConnectorsResource.java | 10 +- .../internal/connectors/FacadeResource.java | 17 +- 20 files changed, 259 insertions(+), 69 deletions(-) diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 33b9ca1f..7cd228b3 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -25,6 +25,7 @@ import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import be.nbb.sdmx.facade.xml.stream.Stax; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.client.Parser; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; @@ -36,6 +37,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; /** @@ -58,7 +60,7 @@ public static final SdmxRepository nbb() { .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dsd = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); - try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA::openReader)) { + try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).parseReader(XIF, SdmxSource.NBB_DATA::openReader)) { result.copyOf(flowRef, cursor); } return result @@ -83,7 +85,7 @@ public static final SdmxRepository ecb() { .filter(o -> o.getRef().equals(flowRef)) .collect(Collectors.toList())); DataStructure dfs = dataStructures.get(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); - try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { + try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).parseReader(XIF, SdmxSource.ECB_DATA::openReader)) { result.copyOf(flowRef, cursor); } return result @@ -111,7 +113,7 @@ private static Dataflow toDataFlow(DataFlowStructure input) { private static T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { XMLEventReader r = null; try { - r = SdmxSource.XIF.createXMLEventReader(xml.openReader()); + r = XIF.createXMLEventReader(xml.openReader()); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); @@ -125,4 +127,6 @@ private static T parse(ByteSource xml, LanguagePriorityList l, Parser par } } } + + private static final XMLInputFactory XIF = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index c7689df7..45addcdb 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.xml.stream.Stax; import internal.file.SdmxFileUtil; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; @@ -41,8 +42,8 @@ public final class SdmxFileManager implements SdmxConnectionSupplier, HasCache { @Nonnull public static SdmxFileManager of() { - XMLInputFactory factory = XMLInputFactory.newInstance(); - return new SdmxFileManager(factory, new XMLStreamSdmxDecoder(factory), new AtomicReference<>(new ConcurrentHashMap())); + XMLInputFactory factory = Stax.getInputFactory(); + return new SdmxFileManager(factory, new XMLStreamSdmxDecoder(factory, Stax.getInputFactoryWithoutNamespace()), new AtomicReference<>(new ConcurrentHashMap())); } private final XMLInputFactory factory; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java index be1a3894..b9b8224b 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java @@ -35,6 +35,10 @@ public static Stax.Parser of() { } private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws XMLStreamException { + if (Stax.isNotNamespaceAware(reader)) { + throw new XMLStreamException("Cannot probe data type"); + } + int level = 0; while (reader.hasNext()) { switch (reader.next()) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 23cc560d..333353da 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -18,11 +18,11 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.file.SdmxFileSet; +import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.File; import java.io.StringReader; import java.io.StringWriter; import javax.annotation.Nonnull; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; @@ -65,7 +65,7 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE String data = null; String structure = null; try { - XMLStreamReader xml = INPUT.createXMLStreamReader(new StringReader(input)); + XMLStreamReader xml = Stax.getInputFactoryWithoutNamespace().createXMLStreamReader(new StringReader(input)); while (xml.hasNext()) { if (xml.next() == XMLStreamReader.START_ELEMENT && xml.getLocalName().equals(ROOT_TAG)) { data = xml.getAttributeValue(null, DATA_ATTR); @@ -86,5 +86,4 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE private static final String DATA_ATTR = "data"; private static final String STRUCT_ATTR = "structure"; private static final XMLOutputFactory OUTPUT = XMLOutputFactory.newInstance(); - private static final XMLInputFactory INPUT = XMLInputFactory.newInstance(); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index 63e68751..e1ab1e57 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -39,6 +39,7 @@ public final class XMLStreamSdmxDecoder implements SdmxDecoder { private final XMLInputFactory factory; + private final XMLInputFactory factoryWithoutNamespace; @Override public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { @@ -70,17 +71,7 @@ private Stax.Parser> getStructParser(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - if (factory.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE) - && (Boolean) factory.getProperty(XMLInputFactory.IS_NAMESPACE_AWARE)) { - factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); - try { - return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); - } finally { - factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true); - } - } else { - return getStructDecoder(dataType).parseFile(factory, data, StandardCharsets.UTF_8); - } + return getStructDecoder(dataType).parseFile(factoryWithoutNamespace, data, StandardCharsets.UTF_8); } private static Stax.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index 16ad9850..6cc4a3a8 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -21,9 +21,10 @@ import java.io.IOException; import org.junit.Test; import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.xml.stream.Stax; import static org.assertj.core.api.Assertions.assertThat; import static internal.file.CustomDataStructureBuilder.dimension; -import static internal.file.CustomDataStructureBuilder.dimension; +import javax.xml.stream.XMLInputFactory; /** * @@ -44,7 +45,7 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic20().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic20().parseReader(xif, SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); } @Test @@ -61,7 +62,7 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact20().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact20().parseReader(xif, SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); } @Test @@ -80,7 +81,7 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic21().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic21().parseReader(xif, SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); } @Test @@ -99,6 +100,8 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact21().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact21().parseReader(xif, SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); } + + private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java index 3379dbd0..47952ed3 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java @@ -17,7 +17,9 @@ package internal.file; import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.IOException; +import javax.xml.stream.XMLInputFactory; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -29,25 +31,27 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { - assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC20::openReader)) + assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_GENERIC20::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC20); } @Test public void testDecodeCompact20() throws IOException { - assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT20::openReader)) + assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_COMPACT20::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT20); } @Test public void testDecodeGeneric21() throws IOException { - assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_GENERIC21::openReader)) + assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_GENERIC21::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC21); } @Test public void testDecodeCompact21() throws IOException { - assertThat(DataTypeProbe.of().parseReader(SdmxSource.XIF, SdmxSource.OTHER_COMPACT21::openReader)) + assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_COMPACT21::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT21); } + + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index f88a2c51..6e1a2f76 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -24,8 +24,10 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; +import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.File; import java.io.IOException; +import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; import org.junit.Rule; @@ -37,10 +39,6 @@ */ public class FileSdmxConnectionTest { - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(SdmxSource.XIF); - @Test public void testCompactData21() throws IOException { File compact21 = temp.newFile(); @@ -48,7 +46,7 @@ public void testCompactData21() throws IOException { SdmxFileSet files = SdmxFileSet.of(compact21, null); - FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, SdmxSource.XIF, decoder); + FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, xif, decoder); assertThat(conn.getFlows()).hasSize(1); assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(files)).getDimensions()).hasSize(7); @@ -76,6 +74,12 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, SdmxSource.XIF, decoder), SdmxFileUtil.asDataflowRef(files)); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, xif, decoder), SdmxFileUtil.asDataflowRef(files)); } + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private final XMLInputFactory xif = Stax.getInputFactory(); + private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(xif, Stax.getInputFactoryWithoutNamespace()); } diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java index 7141403a..0a922688 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/samples/SdmxSource.java @@ -16,8 +16,6 @@ */ package be.nbb.sdmx.facade.samples; -import javax.xml.stream.XMLInputFactory; - /** * * @author Philippe Charles @@ -41,6 +39,4 @@ private static ByteSource of(String name) { public static final ByteSource OTHER_COMPACT20 = of("other/sdmx-compact20-sample.xml"); public static final ByteSource OTHER_GENERIC21 = of("other/sdmx-GenericData21.xml"); public static final ByteSource OTHER_COMPACT21 = of("other/sdmx-compactData21.xml"); - - public static final XMLInputFactory XIF = XMLInputFactory.newInstance(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java index 9637ae4b..b1b7b2e4 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java @@ -27,9 +27,16 @@ import java.nio.file.Files; import java.nio.file.Path; import javax.annotation.Nonnull; +import javax.xml.stream.EventFilter; +import javax.xml.stream.StreamFilter; +import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLReporter; +import javax.xml.stream.XMLResolver; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.util.XMLEventAllocator; +import javax.xml.transform.Source; /** * @@ -133,4 +140,153 @@ void ensureClosed(XMLStreamException ex, Closeable onClose) throws IOException { ex.addSuppressed(suppressed); } } + + @Nonnull + public XMLInputFactory getInputFactory() { + return ImmutableInputFactory.DEFAULT; + } + + @Nonnull + public XMLInputFactory getInputFactoryWithoutNamespace() { + return ImmutableInputFactory.WITHOUT_NAMESPACE; + } + + public boolean isNotNamespaceAware(@Nonnull XMLStreamReader f) { + return !(Boolean) f.getProperty(XMLInputFactory.IS_NAMESPACE_AWARE); + } + + private static final class ImmutableInputFactory extends XMLInputFactory { + + static final XMLInputFactory DEFAULT = new ImmutableInputFactory(true); + static final XMLInputFactory WITHOUT_NAMESPACE = new ImmutableInputFactory(false); + + private final XMLInputFactory delegate; + + private ImmutableInputFactory(boolean namespaceAware) { + this.delegate = XMLInputFactory.newFactory(); + if (!namespaceAware && delegate.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE)) { + delegate.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); + } + } + + @Override + public XMLStreamReader createXMLStreamReader(Reader reader) throws XMLStreamException { + return delegate.createXMLStreamReader(reader); + } + + @Override + public XMLStreamReader createXMLStreamReader(Source source) throws XMLStreamException { + return delegate.createXMLStreamReader(source); + } + + @Override + public XMLStreamReader createXMLStreamReader(InputStream stream) throws XMLStreamException { + return delegate.createXMLStreamReader(stream); + } + + @Override + public XMLStreamReader createXMLStreamReader(InputStream stream, String encoding) throws XMLStreamException { + return delegate.createXMLStreamReader(stream, encoding); + } + + @Override + public XMLStreamReader createXMLStreamReader(String systemId, InputStream stream) throws XMLStreamException { + return delegate.createXMLStreamReader(systemId, stream); + } + + @Override + public XMLStreamReader createXMLStreamReader(String systemId, Reader reader) throws XMLStreamException { + return delegate.createXMLStreamReader(systemId, reader); + } + + @Override + public XMLEventReader createXMLEventReader(Reader reader) throws XMLStreamException { + return delegate.createXMLEventReader(reader); + } + + @Override + public XMLEventReader createXMLEventReader(String systemId, Reader reader) throws XMLStreamException { + return delegate.createXMLEventReader(systemId, reader); + } + + @Override + public XMLEventReader createXMLEventReader(XMLStreamReader reader) throws XMLStreamException { + return delegate.createXMLEventReader(reader); + } + + @Override + public XMLEventReader createXMLEventReader(Source source) throws XMLStreamException { + return delegate.createXMLEventReader(source); + } + + @Override + public XMLEventReader createXMLEventReader(InputStream stream) throws XMLStreamException { + return delegate.createXMLEventReader(stream); + } + + @Override + public XMLEventReader createXMLEventReader(InputStream stream, String encoding) throws XMLStreamException { + return delegate.createXMLEventReader(stream, encoding); + } + + @Override + public XMLEventReader createXMLEventReader(String systemId, InputStream stream) throws XMLStreamException { + return delegate.createXMLEventReader(systemId, stream); + } + + @Override + public XMLStreamReader createFilteredReader(XMLStreamReader reader, StreamFilter filter) throws XMLStreamException { + return delegate.createFilteredReader(reader, filter); + } + + @Override + public XMLEventReader createFilteredReader(XMLEventReader reader, EventFilter filter) throws XMLStreamException { + return delegate.createFilteredReader(reader, filter); + } + + @Override + public XMLResolver getXMLResolver() { + return delegate.getXMLResolver(); + } + + @Override + public void setXMLResolver(XMLResolver resolver) { + throw new UnsupportedOperationException(); + } + + @Override + public XMLReporter getXMLReporter() { + return delegate.getXMLReporter(); + } + + @Override + public void setXMLReporter(XMLReporter reporter) { + throw new UnsupportedOperationException(); + } + + @Override + public void setProperty(String name, Object value) throws IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public Object getProperty(String name) throws IllegalArgumentException { + return delegate.getProperty(name); + } + + @Override + public boolean isPropertySupported(String name) { + return delegate.isPropertySupported(name); + } + + @Override + public void setEventAllocator(XMLEventAllocator allocator) { + throw new UnsupportedOperationException(); + } + + @Override + public XMLEventAllocator getEventAllocator() { + return delegate.getEventAllocator(); + } + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index 1b0318dd..208a938d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -74,6 +74,10 @@ final class XMLStreamStructure20 { @Nonnull public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { + if (Stax.isNotNamespaceAware(reader)) { + throw new XMLStreamException("Cannot parse structure"); + } + List result = new ArrayList<>(); Map> codelists = new HashMap<>(); Map concepts = new HashMap<>(); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index c325e1da..5ca4d491 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -76,6 +76,10 @@ final class XMLStreamStructure21 { @Nonnull public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { + if (Stax.isNotNamespaceAware(reader)) { + throw new XMLStreamException("Cannot parse structure"); + } + List result = new ArrayList<>(); while (nextTags(reader, "")) { switch (reader.getLocalName()) { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 5b030659..1d038369 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -26,10 +26,12 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.IOException; import java.time.LocalDateTime; import java.util.Collections; import java.util.List; +import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -75,8 +77,8 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } } @@ -98,8 +100,8 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(120); @@ -107,4 +109,5 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { } private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java index 0faba5a6..4a02e469 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java @@ -103,7 +103,7 @@ public void testInvalidParseStream() throws IOException { .hasNoCause(); } - private final XMLInputFactory xif = SdmxSource.XIF; + private final XMLInputFactory xif = Stax.getInputFactory(); private static class EmptyStream extends InputStream { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index 4120de1a..aadd5e10 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; import be.nbb.sdmx.facade.util.FreqParser; import java.io.InputStream; +import javax.xml.stream.XMLInputFactory; /** * @@ -42,11 +43,11 @@ public void testCompactData20() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE"); + return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE"); }); try (InputStream stream = xml.openStream(); - DataCursor o = new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { + DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -86,11 +87,11 @@ public void testCompactData21() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE"); + return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE"); }); try (InputStream stream = xml.openStream(); - DataCursor o = new XMLStreamCompactDataCursor(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { + DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -118,4 +119,6 @@ public void testCompactData21() throws Exception { assertThat(o.nextSeries()).isFalse(); } } + + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 183c78b7..7f1e086f 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; import be.nbb.sdmx.facade.util.FreqParser; import java.io.InputStream; +import javax.xml.stream.XMLInputFactory; /** * @@ -42,11 +43,11 @@ public void testGenericData20() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx20(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20()); + return XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20()); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx20(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20())) { + DataCursor o = XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20())) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -90,11 +91,11 @@ public void testGenericData21() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -130,11 +131,11 @@ public void testGenericData21Bis() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx21(SdmxSource.XIF.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -166,4 +167,6 @@ public void testGenericData21Bis() throws Exception { assertThat(indexSeries).isEqualTo(119); } } + + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index 4b2a1dd3..e54a8b0a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import java.util.List; +import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -38,7 +39,7 @@ public class XMLStreamStructure20Test { public void test() throws Exception { Stax.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - assertThat(p1.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p1.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -52,7 +53,7 @@ public void test() throws Exception { Stax.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - assertThat(p2.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p2.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -64,9 +65,11 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> p1.parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader)) + assertThatThrownBy(() -> p1.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)) .isInstanceOf(IOException.class) .hasCauseInstanceOf(XMLStreamException.class) .hasMessageContaining("Invalid namespace"); } + + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index 76b4ced7..f00b31af 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.IOException; import java.util.List; +import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -38,7 +39,7 @@ public class XMLStreamStructure21Test { public void test() throws Exception { Stax.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - assertThat(parser.parseReader(SdmxSource.XIF, SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(parser.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); @@ -50,9 +51,11 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> parser.parseReader(SdmxSource.XIF, SdmxSource.NBB_DATA_STRUCTURE::openReader)) + assertThatThrownBy(() -> parser.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)) .isInstanceOf(IOException.class) .hasCauseInstanceOf(XMLStreamException.class) .hasMessageContaining("Invalid namespace"); } + + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index 51b0189e..81c8ad4d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -24,6 +24,7 @@ import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.xml.stream.Stax; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; @@ -39,6 +40,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; /** @@ -102,7 +104,7 @@ List flow20(ByteSource xml, LanguagePriorityList l) throws IOException List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data20(SdmxSource.XIF, xml, Util.toStructure(dsd)) + return FacadeResource.data20(XIF, xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -118,7 +120,7 @@ List flow21(ByteSource xml, LanguagePriorityList l) throws IOException List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data21(SdmxSource.XIF, xml, Util.toStructure(dsd)) + return FacadeResource.data21(XIF, xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -167,7 +169,7 @@ Dataflow asDataflow(DataFlowStructure o) { T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { XMLEventReader r = null; try { - r = SdmxSource.XIF.createXMLEventReader(xml.openReader()); + r = XIF.createXMLEventReader(xml.openReader()); return parser.parse(r, l); } catch (XMLStreamException | SdmxException ex) { throw new IOException(ex); @@ -206,4 +208,6 @@ private char formatByStandardFreq(Frequency code) { return '?'; } } + + private final XMLInputFactory XIF = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java index f36fbb06..45a38b76 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java @@ -27,6 +27,7 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -40,12 +41,11 @@ public class FacadeResource { public SdmxRepository nbb() throws IOException { - XMLInputFactory f = SdmxSource.XIF; LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct20(f, SdmxSource.NBB_DATA_STRUCTURE, l); - List flows = flow20(f, SdmxSource.NBB_DATA_STRUCTURE, l); - List data = data20(f, SdmxSource.NBB_DATA, structs.get(0)); + List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); DataflowRef ref = DataflowRef.of("NBB", "TEST_DATASET", null); @@ -59,12 +59,11 @@ public SdmxRepository nbb() throws IOException { } public SdmxRepository ecb() throws IOException { - XMLInputFactory f = SdmxSource.XIF; LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct21(f, SdmxSource.ECB_DATA_STRUCTURE, l); - List flows = flow21(f, SdmxSource.ECB_DATAFLOWS, l); - List data = data21(f, SdmxSource.ECB_DATA, structs.get(0)); + List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); + List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); DataflowRef ref = DataflowRef.of("ECB", "AME", "1.0"); @@ -115,4 +114,6 @@ private Dataflow asDataflow(DataStructure o) { DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); return Dataflow.of(ref, o.getRef(), o.getLabel()); } + + private final XMLInputFactory XIF = Stax.getInputFactory(); } From 06f381157f4793b39e587bded85895f847d98d1e Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 14:59:16 +0100 Subject: [PATCH 40/68] Fixed UI freeze when using SdmxFileProvider. --- .../demetra/sdmx/file/SdmxFileProvider.java | 29 +++++++++++++++++-- .../internal/file/FileSdmxConnection.java | 2 +- .../main/java/internal/file/SdmxFileUtil.java | 5 ++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 6b294087..c5101b61 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -29,6 +29,7 @@ import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.DataSource; +import ec.tss.tsproviders.HasDataDisplayName; import ec.tss.tsproviders.HasDataMoniker; import ec.tss.tsproviders.HasDataSourceBean; import ec.tss.tsproviders.HasDataSourceMutableList; @@ -75,7 +76,7 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { @lombok.experimental.Delegate private final HasFilePaths filePathSupport; - @lombok.experimental.Delegate(excludes = HasTsCursor.class) + @lombok.experimental.Delegate(excludes = {HasTsCursor.class, HasDataDisplayName.class}) private final CubeSupport cubeSupport; @lombok.experimental.Delegate @@ -110,6 +111,26 @@ public boolean accept(File pathname) { return pathname.getName().toLowerCase().endsWith(".xml"); } + @Override + public String getDisplayName(DataSource dataSource) throws IllegalArgumentException { + return getSourceLabel(decodeBean(dataSource)); + } + + @Override + public String getDisplayName(DataSet dataSet) throws IllegalArgumentException { + return cubeSupport.getDisplayName(dataSet); + } + + @Override + public String getDisplayName(IOException exception) throws IllegalArgumentException { + return cubeSupport.getDisplayName(exception); + } + + @Override + public String getDisplayNodeName(DataSet dataSet) throws IllegalArgumentException { + return cubeSupport.getDisplayNodeName(dataSet); + } + @lombok.AllArgsConstructor private static final class SdmxCubeResource implements CubeSupport.Resource { @@ -143,7 +164,7 @@ private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), conn, flow); - CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), bean.getFile().getPath()); + CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), getSourceLabel(bean)); IParam idParam = param.getCubeIdParam(accessor.getRoot()); @@ -162,4 +183,8 @@ private static IO.Supplier getSupplier(HasSdmxProperties propert return () -> supplier.getConnection(name, languages); } } + + private static String getSourceLabel(SdmxFileBean bean) { + return bean.getFile().getPath(); + } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 6b895fbd..28852cd0 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -59,7 +59,7 @@ class FileSdmxConnection implements SdmxConnection { this.languages = languages; this.factory = factory; this.decoder = decoder; - this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(files), EMPTY, files.getData().getName().replace(".xml", "")); + this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(files), EMPTY, SdmxFileUtil.asFlowLabel(files)); this.closed = false; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 333353da..6754678b 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -41,6 +41,11 @@ public DataflowRef asDataflowRef(@Nonnull SdmxFileSet files) { return DataflowRef.parse("data" + (structFile != null && !structFile.toString().isEmpty() ? "&struct" : "")); } + @Nonnull + public String asFlowLabel(@Nonnull SdmxFileSet files) { + return files.getData().getName().replace(".xml", ""); + } + @Nonnull public String toXml(@Nonnull SdmxFileSet files) { StringWriter result = new StringWriter(); From 9576142bac24f86718be3faaf22ccd65128f826c Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 16:01:44 +0100 Subject: [PATCH 41/68] Improved xml parsing speed by avoiding Stax namespace parsing. --- .../nbb/sdmx/facade/file/SdmxFileManager.java | 8 +-- .../file/CachedFileSdmxConnection.java | 4 +- .../internal/file/FileSdmxConnection.java | 8 +-- .../internal/file/FileSdmxConnectionTest.java | 8 +-- .../stream/XMLStreamCompactDataCursor.java | 13 +++-- .../stream/XMLStreamGenericDataCursor.java | 50 ++++++++++--------- .../sdmx/facade/xml/stream/XMLStreamUtil.java | 10 ++-- .../nbb/sdmx/facade/xml/stream/StaxTest.java | 25 +++++++++- .../XMLStreamCompactDataCursorTest.java | 2 +- .../XMLStreamGenericDataCursorTest.java | 2 +- 10 files changed, 82 insertions(+), 48 deletions(-) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 45addcdb..c6bc175d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -42,11 +42,11 @@ public final class SdmxFileManager implements SdmxConnectionSupplier, HasCache { @Nonnull public static SdmxFileManager of() { - XMLInputFactory factory = Stax.getInputFactory(); - return new SdmxFileManager(factory, new XMLStreamSdmxDecoder(factory, Stax.getInputFactoryWithoutNamespace()), new AtomicReference<>(new ConcurrentHashMap())); + XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); + return new SdmxFileManager(factoryWithoutNamespace, new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), new AtomicReference<>(new ConcurrentHashMap())); } - private final XMLInputFactory factory; + private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final AtomicReference cache; @@ -65,7 +65,7 @@ public SdmxConnection getConnection(String name, LanguagePriorityList languages) @Nonnull public SdmxConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { - return new CachedFileSdmxConnection(files, languages, factory, decoder, cache.get()); + return new CachedFileSdmxConnection(files, languages, factoryWithoutNamespace, decoder, cache.get()); } @Override diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index f5a40846..179c2b68 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -46,8 +46,8 @@ public final class CachedFileSdmxConnection extends FileSdmxConnection { private final TypedId decodeKey; private final TypedId> loadDataKey; - public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder, ConcurrentMap cache) { - super(files, languages, factory, decoder); + public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, ConcurrentMap cache) { + super(files, languages, factoryWithoutNamespace, decoder); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); String base = SdmxFileUtil.toXml(files) + languages.toString(); this.decodeKey = TypedId.of("decode://" + base); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 28852cd0..3969bf51 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -49,15 +49,15 @@ class FileSdmxConnection implements SdmxConnection { private final SdmxFileSet files; private final LanguagePriorityList languages; - private final XMLInputFactory factory; + private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final Dataflow dataflow; private boolean closed; - FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factory, SdmxDecoder decoder) { + FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder) { this.files = files; this.languages = languages; - this.factory = factory; + this.factoryWithoutNamespace = factoryWithoutNamespace; this.decoder = decoder; this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(files), EMPTY, SdmxFileUtil.asFlowLabel(files)); this.closed = false; @@ -120,7 +120,7 @@ protected SdmxDecoder.Info decode() throws IOException { } protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factory, files.getData(), StandardCharsets.UTF_8); + return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factoryWithoutNamespace, files.getData(), StandardCharsets.UTF_8); } private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 6e1a2f76..2bcd2ff8 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -46,7 +46,7 @@ public void testCompactData21() throws IOException { SdmxFileSet files = SdmxFileSet.of(compact21, null); - FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, xif, decoder); + FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder); assertThat(conn.getFlows()).hasSize(1); assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(files)).getDimensions()).hasSize(7); @@ -74,12 +74,12 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, xif, decoder), SdmxFileUtil.asDataflowRef(files)); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder), SdmxFileUtil.asDataflowRef(files)); } @Rule public TemporaryFolder temp = new TemporaryFolder(); - private final XMLInputFactory xif = Stax.getInputFactory(); - private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(xif, Stax.getInputFactoryWithoutNamespace()); + private final XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); + private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index c0f94230..8279950a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -30,7 +30,9 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.util.FreqParser; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; +import java.util.logging.Logger; /** * @@ -55,6 +57,9 @@ final class XMLStreamCompactDataCursor implements DataCursor { private boolean hasObs; XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, String timeDimensionId, String primaryMeasureId) { + if (!Stax.isNotNamespaceAware(reader)) { + Logger.getLogger(getClass().getName()).warning("Using XMLStreamReader with namespace awareness"); + } this.reader = reader; this.onClose = onClose; this.keyBuilder = keyBuilder; @@ -158,9 +163,9 @@ private void checkObsState() throws IOException, IllegalStateException { private Status onDataSet(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(SERIES_TAG) ? parseSeries() : CONTINUE; + return isTagMatch(localName, SERIES_TAG) ? parseSeries() : CONTINUE; } else { - return localName.equals(DATASET_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, DATASET_TAG) ? HALT : CONTINUE; } } @@ -183,9 +188,9 @@ private void parserSerieHead() { private Status onSeriesBody(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(OBS_TAG) ? parseObs() : CONTINUE; + return isTagMatch(localName, OBS_TAG) ? parseObs() : CONTINUE; } else { - return localName.equals(SERIES_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, SERIES_TAG) ? HALT : CONTINUE; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 409164f4..63273f20 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -30,9 +30,11 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.util.FreqParser; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.logging.Logger; import javax.annotation.Nonnull; /** @@ -70,6 +72,9 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClo private boolean hasObs; private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { + if (!Stax.isNotNamespaceAware(reader)) { + Logger.getLogger(getClass().getName()).warning("Using XMLStreamReader with namespace awareness"); + } this.reader = reader; this.onClose = onClose; this.keyBuilder = keyBuilder; @@ -179,9 +184,9 @@ private void checkObsState() throws IOException, IllegalStateException { private Status onDataSet(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(SERIES_TAG) ? parseSeries() : CONTINUE; + return isTagMatch(localName, SERIES_TAG) ? parseSeries() : CONTINUE; } else { - return localName.equals(DATASET_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, DATASET_TAG) ? HALT : CONTINUE; } } @@ -193,18 +198,17 @@ private Status parseSeries() throws XMLStreamException { private Status onSeriesHead(boolean start, String localName) throws XMLStreamException { if (start) { - switch (localName) { - case SERIES_KEY_TAG: - return parseSeriesKey(); - case ATTRIBUTES_TAG: - return parseAttributes(); - case OBS_TAG: - return HALT; - default: - return CONTINUE; + if (isTagMatch(localName, SERIES_KEY_TAG)) { + return parseSeriesKey(); + } else if (isTagMatch(localName, ATTRIBUTES_TAG)) { + return parseAttributes(); + } else if (isTagMatch(localName, OBS_TAG)) { + return HALT; + } else { + return CONTINUE; } } else { - return localName.equals(SERIES_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, SERIES_TAG) ? HALT : CONTINUE; } } @@ -220,17 +224,17 @@ private Status parseAttributes() throws XMLStreamException { private Status onSeriesKey(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(VALUE_TAG) ? parseSeriesKeyValue() : CONTINUE; + return isTagMatch(localName, VALUE_TAG) ? parseSeriesKeyValue() : CONTINUE; } else { - return localName.equals(SERIES_KEY_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, SERIES_KEY_TAG) ? HALT : CONTINUE; } } private Status onAttributes(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(VALUE_TAG) ? parseAttributesValue() : CONTINUE; + return isTagMatch(localName, VALUE_TAG) ? parseAttributesValue() : CONTINUE; } else { - return localName.equals(ATTRIBUTES_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, ATTRIBUTES_TAG) ? HALT : CONTINUE; } } @@ -245,29 +249,29 @@ private Status parseAttributesValue() throws XMLStreamException { } private boolean isCurrentElementStartOfObs() { - return reader.getEventType() == XMLStreamReader.START_ELEMENT && OBS_TAG.equals(reader.getLocalName()); + return reader.isStartElement() && isTagMatch(reader.getLocalName(), OBS_TAG); } private boolean isCurrentElementEnfOfSeries() { - return reader.getEventType() == XMLStreamReader.END_ELEMENT && SERIES_TAG.equals(reader.getLocalName()); + return reader.isEndElement() && isTagMatch(reader.getLocalName(), SERIES_TAG); } private Status onSeriesBody(boolean start, String localName) throws XMLStreamException { if (start) { - return localName.equals(OBS_TAG) ? parseObs() : CONTINUE; + return isTagMatch(localName, OBS_TAG) ? parseObs() : CONTINUE; } else { - return localName.equals(SERIES_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, SERIES_TAG) ? HALT : CONTINUE; } } private Status onObs(boolean start, String localName) throws XMLStreamException { if (start) { - if (localName.equals(headParser.getTimeELement())) { + if (isTagMatch(localName, headParser.getTimeELement())) { return parseObsTime(); } - return localName.equals(OBS_VALUE_TAG) ? parseObsValue() : CONTINUE; + return isTagMatch(localName, OBS_VALUE_TAG) ? parseObsValue() : CONTINUE; } else { - return localName.equals(OBS_TAG) ? HALT : CONTINUE; + return isTagMatch(localName, OBS_TAG) ? HALT : CONTINUE; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java index 45377d03..c7c5917e 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java @@ -69,7 +69,7 @@ static boolean nextTags(XMLStreamReader reader, String tag) throws XMLStreamExce case XMLStreamReader.START_ELEMENT: return true; case XMLStreamReader.END_ELEMENT: - if (tag.equals(reader.getLocalName())) { + if (isTagMatch(reader.getLocalName(), tag)) { return false; } break; @@ -82,12 +82,12 @@ static boolean nextTag(XMLStreamReader reader, String end, String start) throws while (reader.hasNext()) { switch (reader.next()) { case XMLStreamReader.START_ELEMENT: - if (start.equals(reader.getLocalName())) { + if (isTagMatch(reader.getLocalName(), start)) { return true; } break; case XMLStreamReader.END_ELEMENT: - if (end.equals(reader.getLocalName())) { + if (isTagMatch(reader.getLocalName(), end)) { return false; } break; @@ -105,4 +105,8 @@ static void check(boolean expression, Supplier location, String messag throw new XMLStreamException(String.format(message, args), location.get()); } } + + static boolean isTagMatch(String localName, String tag) { + return localName.endsWith(tag) && (localName.length() == tag.length() || localName.charAt(localName.length() - 1 - tag.length()) == ':'); + } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java index 4a02e469..01b19f9f 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java @@ -16,14 +16,13 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.sdmx.facade.samples.SdmxSource; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import static java.nio.charset.StandardCharsets.UTF_8; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -32,6 +31,28 @@ */ public class StaxTest { + @Test + public void testIsTagMatch() { + assertThat(XMLStreamUtil.isTagMatch("", "")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "HelloWorld")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "helloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("helloWorld", "HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("", "HelloWorld")).isFalse(); + + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "HelloWorld")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "helloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "Hello")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "World")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "")).isFalse(); + + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("helloWorld", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("Hello", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("World", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("", "xxx:HelloWorld")).isFalse(); + } + @Test @SuppressWarnings("null") public void testValidParseStream() throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index aadd5e10..84e5134d 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -120,5 +120,5 @@ public void testCompactData21() throws Exception { } } - private final XMLInputFactory xif = Stax.getInputFactory(); + private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 7f1e086f..12077a2f 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -168,5 +168,5 @@ public void testGenericData21Bis() throws Exception { } } - private final XMLInputFactory xif = Stax.getInputFactory(); + private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); } From 40c24e9d7e015bc22bfd48a0e0335887eda0510c Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 16 Nov 2017 17:22:40 +0100 Subject: [PATCH 42/68] Fixed compact20 decoding NPE and missing values. --- .../src/main/java/internal/file/DataStructureDecoder.java | 5 ++--- .../test/java/internal/file/DataStructureDecoderTest.java | 4 ++-- .../java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index 3fca3d61..4d8b5a3d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -142,12 +142,11 @@ private static void generic20Attributes(XMLStreamReader reader, CustomDataStruct // private static DataStructure compact20(XMLStreamReader reader) throws XMLStreamException { CustomDataStructureBuilder builder = new CustomDataStructureBuilder().fileType(COMPACT20); + builder.refId("UNKNOWN"); // FIXME: find a way to parse/guess this information while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: - if (isTagMatch(reader, "CompactData")) { - builder.refId("TODO"); - } else if (isTagMatch(reader, "DataSet")) { + if (isTagMatch(reader, "DataSet")) { compact20DataSet(reader, builder); } break; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java index 6cc4a3a8..fb2b454c 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java @@ -51,13 +51,13 @@ public void testDecodeGeneric20() throws IOException { @Test public void testDecodeCompact20() throws IOException { DataStructure ds = DataStructure.builder() - .ref(DataStructureRef.of(null, "TODO", null)) + .ref(DataStructureRef.of(null, "UNKNOWN", null)) .dimension(dimension("FREQ", 1, "A", "M")) .dimension(dimension("COLLECTION", 2, "B")) .dimension(dimension("VIS_CTY", 3, "MX")) .dimension(dimension("JD_TYPE", 4, "P")) .dimension(dimension("JD_CATEGORY", 5, "A", "B")) - .label("TODO") + .label("UNKNOWN") .timeDimensionId("TIME_PERIOD") .primaryMeasureId("OBS_VALUE") .build(); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index 2350e20e..a6c1262b 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -39,7 +39,7 @@ public Stax.Parser compactData20(@Nonnull DataStructure dsd) throws @Nonnull public Stax.Parser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), "", ""); + return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull From eeb4c019af2d3ece713bd647d101cfb7a7cdd9a9 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 17 Nov 2017 10:38:48 +0100 Subject: [PATCH 43/68] Improved use of SdmxFileSet. --- .../be/nbb/demetra/sdmx/file/SdmxFileProvider.java | 2 +- .../demetra/sdmx/file/SdmxFileProviderBuddy.java | 3 ++- .../java/be/nbb/sdmx/facade/file/SdmxFileSet.java | 11 +++++++++++ .../main/java/internal/file/FileSdmxConnection.java | 2 +- .../src/main/java/internal/file/SdmxFileUtil.java | 13 +++---------- .../java/internal/file/XMLStreamSdmxDecoder.java | 2 +- .../be/nbb/sdmx/facade/file/SdmxFileSetTest.java | 12 ++++++++++++ .../java/internal/file/FileSdmxConnectionTest.java | 6 +++--- .../test/java/internal/file/SdmxFileUtilTest.java | 12 ------------ 9 files changed, 34 insertions(+), 29 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index c5101b61..b7da3197 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -158,7 +158,7 @@ private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths SdmxFileBean bean = param.get(dataSource); SdmxFileSet files = SdmxCubeItems.resolveFileSet(paths, bean); - DataflowRef flow = SdmxFileUtil.asDataflowRef(files); + DataflowRef flow = files.asDataflowRef(); IO.Supplier conn = getSupplier(properties, files); diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java index 2bf1e274..d7ffc3e2 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java @@ -20,6 +20,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.file.SdmxFileManager; +import be.nbb.sdmx.facade.file.SdmxFileSet; import com.google.common.base.Converter; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -220,7 +221,7 @@ private NodePropertySetBuilder withOptions(NodePropertySetBuilder b, SdmxFileBea .add(); Supplier toSource = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileUtil::toXml).orElse(""); - Supplier toFlow = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileUtil::asDataflowRef).map(Object::toString).orElse(""); + Supplier toFlow = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileSet::asDataflowRef).map(Object::toString).orElse(""); b.withAutoCompletion() .select(bean, "dimensions", List.class, Joiner.on(',')::join, Splitter.on(',').trimResults().omitEmptyStrings()::splitToList) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java index 32695001..897a2a19 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java @@ -16,7 +16,9 @@ */ package be.nbb.sdmx.facade.file; +import be.nbb.sdmx.facade.DataflowRef; import java.io.File; +import javax.annotation.Nonnull; import javax.annotation.Nullable; /** @@ -31,4 +33,13 @@ public class SdmxFileSet { @Nullable File structure; + + public boolean hasStructure() { + return structure != null; + } + + @Nonnull + public DataflowRef asDataflowRef() { + return DataflowRef.parse("data" + (structure != null && !structure.toString().isEmpty() ? "&struct" : "")); + } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 3969bf51..c6168e68 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -59,7 +59,7 @@ class FileSdmxConnection implements SdmxConnection { this.languages = languages; this.factoryWithoutNamespace = factoryWithoutNamespace; this.decoder = decoder; - this.dataflow = Dataflow.of(SdmxFileUtil.asDataflowRef(files), EMPTY, SdmxFileUtil.asFlowLabel(files)); + this.dataflow = Dataflow.of(files.asDataflowRef(), EMPTY, SdmxFileUtil.asFlowLabel(files)); this.closed = false; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 6754678b..d214fcdc 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -16,7 +16,6 @@ */ package internal.file; -import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.File; @@ -35,27 +34,21 @@ @lombok.experimental.UtilityClass public class SdmxFileUtil { - @Nonnull - public DataflowRef asDataflowRef(@Nonnull SdmxFileSet files) { - File structFile = files.getStructure(); - return DataflowRef.parse("data" + (structFile != null && !structFile.toString().isEmpty() ? "&struct" : "")); - } - @Nonnull public String asFlowLabel(@Nonnull SdmxFileSet files) { return files.getData().getName().replace(".xml", ""); } @Nonnull + @SuppressWarnings("null") public String toXml(@Nonnull SdmxFileSet files) { StringWriter result = new StringWriter(); try { XMLStreamWriter xml = OUTPUT.createXMLStreamWriter(result); xml.writeEmptyElement(ROOT_TAG); xml.writeAttribute(DATA_ATTR, files.getData().toString()); - File structure = files.getStructure(); - if (structure != null) { - xml.writeAttribute(STRUCT_ATTR, structure.toString()); + if (files.hasStructure()) { + xml.writeAttribute(STRUCT_ATTR, files.getStructure().toString()); } xml.writeEndDocument(); xml.close(); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java index e1ab1e57..1510abae 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java @@ -44,7 +44,7 @@ public final class XMLStreamSdmxDecoder implements SdmxDecoder { @Override public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { DataType dataType = probeDataType(files.getData()); - return Info.of(dataType, files.getStructure() != null + return Info.of(dataType, files.hasStructure() ? parseStruct(dataType, langs, files.getStructure()) : decodeStruct(dataType, files.getData())); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java index 2f93ed1b..fd497b40 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java @@ -41,6 +41,18 @@ public void testFactory() { .hasFieldOrPropertyWithValue("structure", structure); } + @Test + public void testAsDataflowRef() { + assertThat(SdmxFileSet.of(data, structure).asDataflowRef().toString()) + .isEqualTo("all,data&struct,latest"); + + assertThat(SdmxFileSet.of(data, new File("")).asDataflowRef().toString()) + .isEqualTo("all,data,latest"); + + assertThat(SdmxFileSet.of(data, null).asDataflowRef().toString()) + .isEqualTo("all,data,latest"); + } + private final File data = new File("a.xml"); private final File structure = new File("b.xml"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 2bcd2ff8..61e38d4c 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -49,11 +49,11 @@ public void testCompactData21() throws IOException { FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder); assertThat(conn.getFlows()).hasSize(1); - assertThat(conn.getStructure(SdmxFileUtil.asDataflowRef(files)).getDimensions()).hasSize(7); + assertThat(conn.getStructure(files.asDataflowRef()).getDimensions()).hasSize(7); Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getCursor(SdmxFileUtil.asDataflowRef(files), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getCursor(files.asDataflowRef(), DataQuery.of(Key.ALL, false))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -74,7 +74,7 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder), SdmxFileUtil.asDataflowRef(files)); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder), files.asDataflowRef()); } @Rule diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java index c172d16e..34b623e2 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java @@ -28,18 +28,6 @@ */ public class SdmxFileUtilTest { - @Test - public void testAsDataflowRef() { - assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, structure)).toString()) - .isEqualTo("all,data&struct,latest"); - - assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, new File(""))).toString()) - .isEqualTo("all,data,latest"); - - assertThat(SdmxFileUtil.asDataflowRef(SdmxFileSet.of(data, null)).toString()) - .isEqualTo("all,data,latest"); - } - @Test public void testToXml() { assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, structure))) From 0b79cd350e13134926384720cb0d8f414bcdc171 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Fri, 17 Nov 2017 11:17:30 +0100 Subject: [PATCH 44/68] Added SdmxFileConnection to simplify file use case. --- .../demetra/sdmx/file/SdmxFileProvider.java | 6 +- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 6 +- .../java/internal/sdmx/SdmxCubeItems.java | 30 +++++---- .../sdmx/facade/file/SdmxFileConnection.java | 63 +++++++++++++++++++ .../nbb/sdmx/facade/file/SdmxFileManager.java | 15 ++++- .../internal/file/FileSdmxConnection.java | 34 +++++++++- .../internal/file/FileSdmxConnectionTest.java | 21 ++++++- 7 files changed, 150 insertions(+), 25 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index b7da3197..4fb01704 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -160,9 +160,9 @@ private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths DataflowRef flow = files.asDataflowRef(); - IO.Supplier conn = getSupplier(properties, files); + IO.Supplier conn = toConnection(properties, files); - CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), conn, flow); + CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), () -> SdmxCubeItems.loadStructure(conn, flow)); CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), getSourceLabel(bean)); @@ -171,7 +171,7 @@ private static SdmxCubeItems of(HasSdmxProperties properties, HasFilePaths paths return new SdmxCubeItems(accessor, idParam); } - private static IO.Supplier getSupplier(HasSdmxProperties properties, SdmxFileSet files) { + private static IO.Supplier toConnection(HasSdmxProperties properties, SdmxFileSet files) { SdmxConnectionSupplier supplier = properties.getConnectionSupplier(); LanguagePriorityList languages = properties.getLanguages(); diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index a6ce365a..4e600f54 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -135,9 +135,9 @@ private static SdmxCubeItems of(HasSdmxProperties properties, SdmxWebParam param DataflowRef flow = DataflowRef.parse(bean.getFlow()); - IO.Supplier conn = getSupplier(properties, bean.getSource()); + IO.Supplier conn = toConnection(properties, bean.getSource()); - CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), conn, flow); + CubeId root = SdmxCubeItems.getOrLoadRoot(bean.getDimensions(), () -> SdmxCubeItems.loadStructure(conn, flow)); CubeAccessor accessor = SdmxCubeAccessor.of(conn, flow, root, bean.getLabelAttribute(), bean.getSource()) .bulk(bean.getCacheDepth(), GuavaCaches.ttlCacheAsMap(bean.getCacheTtl())); @@ -147,7 +147,7 @@ private static SdmxCubeItems of(HasSdmxProperties properties, SdmxWebParam param return new SdmxCubeItems(accessor, idParam); } - private static IO.Supplier getSupplier(HasSdmxProperties properties, String name) { + private static IO.Supplier toConnection(HasSdmxProperties properties, String name) { SdmxConnectionSupplier supplier = properties.getConnectionSupplier(); LanguagePriorityList languages = properties.getLanguages(); diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index 924891ad..c5d3c679 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -17,6 +17,7 @@ package internal.sdmx; import be.nbb.demetra.sdmx.file.SdmxFileBean; +import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.SdmxConnection; @@ -45,25 +46,28 @@ public class SdmxCubeItems { CubeAccessor accessor; IParam idParam; - public static CubeId getOrLoadRoot(List dimensions, IO.Supplier supplier, DataflowRef flow) throws IOException { - return dimensions.isEmpty() - ? CubeId.root(loadDefaultDimIds(supplier, flow)) - : CubeId.root(dimensions); - } - - public static List loadDefaultDimIds(IO.Supplier supplier, DataflowRef flow) throws IOException { + public static DataStructure loadStructure(IO.Supplier supplier, DataflowRef flow) throws IOException { try (SdmxConnection conn = supplier.getWithIO()) { - return conn - .getStructure(flow) - .getDimensions() - .stream() - .map(Dimension::getId) - .collect(Collectors.toList()); + return conn.getStructure(flow); } catch (RuntimeException ex) { throw new UnexpectedIOException(ex); } } + public static CubeId getOrLoadRoot(List dimensions, IO.Supplier structure) throws IOException { + return dimensions.isEmpty() + ? CubeId.root(loadDefaultDimIds(structure)) + : CubeId.root(dimensions); + } + + public static List loadDefaultDimIds(IO.Supplier structure) throws IOException { + return structure.getWithIO() + .getDimensions() + .stream() + .map(Dimension::getId) + .collect(Collectors.toList()); + } + public static Optional tryResolveFileSet(HasFilePaths paths, SdmxFileBean bean) { try { return Optional.of(resolveFileSet(paths, bean)); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java new file mode 100644 index 00000000..1033b8d8 --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java @@ -0,0 +1,63 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.file; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.Series; +import java.io.IOException; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +public interface SdmxFileConnection extends SdmxConnection { + + @Nonnull + DataflowRef getDataflowRef() throws IOException; + + @Nonnull + default Dataflow getFlow() throws IOException { + DataflowRef ref = getDataflowRef(); + return getFlows() + .stream() + .filter(o -> o.getRef().equals(ref)) + .findFirst() + .orElseThrow(IOException::new); + } + + @Nonnull + default DataStructure getStructure() throws IOException { + return getStructure(getDataflowRef()); + } + + @Nonnull + default DataCursor getCursor(@Nonnull DataQuery query) throws IOException { + return getCursor(getDataflowRef(), query); + } + + @Nonnull + default Stream getStream(@Nonnull DataQuery query) throws IOException { + return getStream(getDataflowRef(), query); + } +} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index c6bc175d..13dff468 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -19,7 +19,6 @@ import internal.file.CachedFileSdmxConnection; import internal.file.SdmxDecoder; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; @@ -51,7 +50,12 @@ public static SdmxFileManager of() { private final AtomicReference cache; @Override - public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { + public SdmxFileConnection getConnection(String name) throws IOException { + return getConnection(name, LanguagePriorityList.ANY); + } + + @Override + public SdmxFileConnection getConnection(String name, LanguagePriorityList languages) throws IOException { SdmxFileSet files; try { @@ -64,7 +68,12 @@ public SdmxConnection getConnection(String name, LanguagePriorityList languages) } @Nonnull - public SdmxConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { + public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files) throws IOException { + return getConnection(files, LanguagePriorityList.ANY); + } + + @Nonnull + public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { return new CachedFileSdmxConnection(files, languages, factoryWithoutNamespace, decoder, cache.get()); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index c6168e68..602ac8d0 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -25,8 +25,8 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.DataQueryDetail; import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.file.SdmxFileConnection; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; @@ -43,7 +43,7 @@ * * @author Philippe Charles */ -class FileSdmxConnection implements SdmxConnection { +class FileSdmxConnection implements SdmxFileConnection { private static final DataStructureRef EMPTY = DataStructureRef.of("", "", ""); @@ -63,12 +63,24 @@ class FileSdmxConnection implements SdmxConnection { this.closed = false; } + @Override + final public DataflowRef getDataflowRef() throws IOException { + checkState(); + return dataflow.getRef(); + } + @Override final public Set getFlows() throws IOException { checkState(); return Collections.singleton(dataflow); } + @Override + final public Dataflow getFlow() throws IOException { + checkState(); + return dataflow; + } + @Override final public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); @@ -77,6 +89,12 @@ final public Dataflow getFlow(DataflowRef flowRef) throws IOException { return dataflow; } + @Override + final public DataStructure getStructure() throws IOException { + checkState(); + return decode().getDataStructure(); + } + @Override final public DataStructure getStructure(DataflowRef flowRef) throws IOException { checkState(); @@ -85,6 +103,13 @@ final public DataStructure getStructure(DataflowRef flowRef) throws IOException return decode().getDataStructure(); } + @Override + final public DataCursor getCursor(DataQuery query) throws IOException { + checkState(); + Objects.requireNonNull(query); + return loadData(decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); + } + @Override final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); @@ -94,6 +119,11 @@ final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws I return loadData(decode(), flowRef, query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); } + @Override + public Stream getStream(DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getCursor(query)); + } + @Override public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { return SeriesSupport.asStream(() -> getCursor(flowRef, query)); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 61e38d4c..23bee8f2 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -21,6 +21,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; @@ -28,7 +29,7 @@ import java.io.File; import java.io.IOException; import javax.xml.stream.XMLInputFactory; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -39,6 +40,24 @@ */ public class FileSdmxConnectionTest { + @Test + @SuppressWarnings("null") + public void testFile() throws IOException { + File compact21 = temp.newFile(); + SdmxSource.OTHER_COMPACT21.copyTo(compact21); + + SdmxFileSet files = SdmxFileSet.of(compact21, null); + + FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder); + + assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); + assertThat(conn.getFlow()).isEqualTo(conn.getFlow(files.asDataflowRef())); + assertThat(conn.getStructure()).isEqualTo(conn.getStructure(files.asDataflowRef())); + assertThatThrownBy(() -> conn.getCursor(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> conn.getStream(null)).isInstanceOf(NullPointerException.class); + assertThat(conn.getStream(DataQuery.of(Key.ALL, false))).containsExactly(conn.getStream(DataQuery.of(Key.ALL, false)).toArray(Series[]::new)); + } + @Test public void testCompactData21() throws IOException { File compact21 = temp.newFile(); From 3941eb4383d5b6b1f3f76105f3910c531c90c34a Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 20 Nov 2017 11:20:04 +0100 Subject: [PATCH 45/68] Added initial support of dialects. --- .../sdmx/file/SdmxFileProviderTest.java | 2 +- .../internal/file/DataStructureDecoder.java | 2 +- .../facade/{util => parser}/DataFactory.java | 18 +++-- .../facade/{util => parser}/FreqParser.java | 4 +- .../facade/{util => parser}/FreqUtil.java | 48 ++++++++++---- .../facade/{util => parser}/ObsParser.java | 5 +- .../sdmx/facade/parser/spi/SdmxDialect.java | 35 ++++++++++ .../be/nbb/sdmx/facade/util/SafeParser.java | 6 -- .../facade/xml/stream/SdmxXmlStreams.java | 11 ++-- .../stream/XMLStreamCompactDataCursor.java | 4 +- .../stream/XMLStreamGenericDataCursor.java | 4 +- .../java/internal/parser/DataFactories.java | 65 +++++++++++++++++++ .../{util => parser}/FreqParsers.java | 4 +- .../main/java/internal/util/SafeParsers.java | 25 ------- .../internal/util/StandardDataFactory.java | 51 --------------- .../FreqUtilTest.java} | 6 +- .../{util => parser}/ObsParserTest.java | 2 +- .../XMLStreamCompactDataCursorTest.java | 6 +- .../XMLStreamGenericDataCursorTest.java | 6 +- .../connectors/GenericSDMXClientResource.java | 2 +- .../connectors/PortableTimeSeriesCursor.java | 6 +- .../connectors/drivers/InseeDriver.java | 12 ++-- ...nseeDataFactory.java => InseeDialect.java} | 32 ++++++--- .../connectors/ConnectorsResource.java | 2 +- .../PortableTimeSeriesCursorTest.java | 2 +- 25 files changed, 216 insertions(+), 144 deletions(-) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/{util => parser}/DataFactory.java (69%) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/{util => parser}/FreqParser.java (96%) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/{util => parser}/FreqUtil.java (72%) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/{util => parser}/ObsParser.java (90%) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/spi/SdmxDialect.java create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java rename sdmx-facade/sdmx-facade-util/src/main/java/internal/{util => parser}/FreqParsers.java (94%) delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/{util/FrequencyUtilTest.java => parser/FreqUtilTest.java} (93%) rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/{util => parser}/ObsParserTest.java (99%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/{InseeDataFactory.java => InseeDialect.java} (80%) diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java index cfd42794..ee8d9eb7 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; -import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; import ec.tss.TsCollectionInformation; import ec.tss.TsInformation; import ec.tss.TsInformationType; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java index 4d8b5a3d..8ef19833 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.DataStructure; import static internal.file.SdmxDecoder.DataType.*; -import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java similarity index 69% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java index 02456ad6..5e9e27a7 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/DataFactory.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java @@ -14,31 +14,39 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; +import be.nbb.sdmx.facade.util.SafeParser; import be.nbb.sdmx.facade.DataStructure; -import internal.util.StandardDataFactory; +import be.nbb.sdmx.facade.Frequency; +import internal.parser.DataFactories; +import java.time.LocalDateTime; import javax.annotation.Nonnull; +import javax.annotation.concurrent.ThreadSafe; /** * * @author Philippe Charles */ +@ThreadSafe public interface DataFactory { @Nonnull FreqParser getFreqParser(@Nonnull DataStructure dsd); @Nonnull - ObsParser getObsParser(@Nonnull DataStructure dsd); + SafeParser getPeriodParser(@Nonnull Frequency freq); + + @Nonnull + SafeParser getValueParser(); @Nonnull static DataFactory sdmx20() { - return StandardDataFactory.SDMX20; + return DataFactories.SDMX20; } @Nonnull static DataFactory sdmx21() { - return StandardDataFactory.SDMX21; + return DataFactories.SDMX21; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java similarity index 96% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java index a9c4de69..8a3fc7df 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java @@ -14,12 +14,12 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import internal.util.FreqParsers; +import internal.parser.FreqParsers; import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java similarity index 72% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java index f7fabbb6..11734fb6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/FreqUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java @@ -14,23 +14,19 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; +import be.nbb.sdmx.facade.util.SafeParser; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Frequency; -import static be.nbb.sdmx.facade.Frequency.DAILY; -import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; -import static be.nbb.sdmx.facade.Frequency.HOURLY; -import static be.nbb.sdmx.facade.Frequency.MINUTELY; -import static be.nbb.sdmx.facade.Frequency.MONTHLY; -import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import static be.nbb.sdmx.facade.Frequency.UNDEFINED; -import static be.nbb.sdmx.facade.Frequency.WEEKLY; +import static be.nbb.sdmx.facade.Frequency.*; import javax.annotation.Nonnull; -import static be.nbb.sdmx.facade.Frequency.ANNUAL; -import static be.nbb.sdmx.facade.Frequency.DAILY_BUSINESS; import be.nbb.sdmx.facade.Key; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.EnumMap; +import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; @@ -80,12 +76,12 @@ public BiFunction, String> extractorByInde public Frequency parseByFreq(@Nonnull String code) { switch (code.length()) { case 0: - return Frequency.UNDEFINED; + return UNDEFINED; case 1: return parseByStandardFreq(code.charAt(0)); default: Frequency base = parseByStandardFreq(code.charAt(0)); - return isMultiplier(code.substring(1)) ? base : Frequency.UNDEFINED; + return isMultiplier(code.substring(1)) ? base : UNDEFINED; } } @@ -149,4 +145,30 @@ public Frequency parseByTimeFormat(@Nonnull String code) { return UNDEFINED; } } + + @Nonnull + public static SafeParser onStandardFreq(@Nonnull Frequency freq) { + return STANDARD_PARSERS.get(freq); + } + + private final Map> STANDARD_PARSERS = initStandardParsers(); + + private Map> initStandardParsers() { + SafeParser yearMonth = SafeParser.onDatePattern("yyyy-MM"); + SafeParser yearMonthDay = SafeParser.onDatePattern("yyyy-MM-dd"); + + Map> result = new EnumMap<>(Frequency.class); + result.put(ANNUAL, SafeParser.onDatePattern("yyyy").or(SafeParser.onDatePattern("yyyy'-01'")).or(SafeParser.onDatePattern("yyyy'-A1'"))); + result.put(HALF_YEARLY, SafeParser.onYearFreqPos("S", 2).or(yearMonth)); + result.put(QUARTERLY, SafeParser.onYearFreqPos("Q", 4).or(yearMonth)); + result.put(MONTHLY, SafeParser.onYearFreqPos("M", 12).or(yearMonth)); + result.put(WEEKLY, yearMonthDay); + result.put(DAILY, yearMonthDay); + // FIXME: needs other pattern for time + result.put(HOURLY, yearMonthDay); + result.put(DAILY_BUSINESS, yearMonthDay); + result.put(MINUTELY, yearMonthDay); + result.put(UNDEFINED, yearMonth); + return Collections.unmodifiableMap(result); + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java similarity index 90% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java index c2488119..9724e406 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/ObsParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java @@ -14,8 +14,9 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; +import be.nbb.sdmx.facade.util.SafeParser; import be.nbb.sdmx.facade.Frequency; import java.time.LocalDateTime; import java.util.function.Function; @@ -30,7 +31,7 @@ public final class ObsParser { @Nonnull public static ObsParser standard() { - return new ObsParser(SafeParser::onStandardFreq, SafeParser.onStandardDouble()); + return new ObsParser(FreqUtil::onStandardFreq, SafeParser.onStandardDouble()); } private final Function> toPeriodParser; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/spi/SdmxDialect.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/spi/SdmxDialect.java new file mode 100644 index 00000000..98e267ff --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/spi/SdmxDialect.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.parser.spi; + +import be.nbb.sdmx.facade.parser.DataFactory; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.ThreadSafe; + +/** + * + * @author Philippe Charles + */ +@ThreadSafe +public interface SdmxDialect extends DataFactory { + + @Nonnull + String getName(); + + @Nonnull + String getDescription(); +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java index f16fee64..db44e052 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java @@ -16,7 +16,6 @@ */ package be.nbb.sdmx.facade.util; -import be.nbb.sdmx.facade.Frequency; import internal.util.SafeParsers; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -72,11 +71,6 @@ static SafeParser onYearFreqPos(@Nonnull String freqCode, @Nonneg return new SafeParsers.YearFreqPos(freqCode, freq); } - @Nonnull - static SafeParser onStandardFreq(@Nonnull Frequency freq) { - return SafeParsers.STANDARD_PARSERS.get(freq); - } - @Nonnull static SafeParser onStandardDouble() { return SafeParsers::doubleOrNull; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index a6c1262b..f02c1090 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -16,11 +16,12 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.sdmx.facade.util.DataFactory; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.parser.ObsParser; import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; @@ -39,7 +40,7 @@ public Stax.Parser compactData20(@Nonnull DataStructure dsd) throws @Nonnull public Stax.Parser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull @@ -49,7 +50,7 @@ public Stax.Parser compactData21(@Nonnull DataStructure dsd) throws @Nonnull public Stax.Parser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); } @Nonnull @@ -59,7 +60,7 @@ public Stax.Parser genericData20(@Nonnull DataStructure dsd) throws @Nonnull public Stax.Parser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return (o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd)); } @Nonnull @@ -69,7 +70,7 @@ public Stax.Parser genericData21(@Nonnull DataStructure dsd) throws @Nonnull public Stax.Parser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), df.getObsParser(dsd), df.getFreqParser(dsd)); + return (o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd)); } @Nonnull diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 8279950a..063ba517 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status.CONTINUE; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status.HALT; @@ -29,7 +29,7 @@ import java.util.Map; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.parser.FreqParser; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; import java.util.logging.Logger; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 63273f20..666503db 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status.CONTINUE; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.Status.HALT; @@ -29,7 +29,7 @@ import java.util.Map; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.parser.FreqParser; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; import java.util.function.BiConsumer; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java new file mode 100644 index 00000000..32907d91 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java @@ -0,0 +1,65 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.parser; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.parser.DataFactory; +import be.nbb.sdmx.facade.parser.FreqParser; +import be.nbb.sdmx.facade.parser.FreqUtil; +import be.nbb.sdmx.facade.util.SafeParser; +import java.time.LocalDateTime; + +/** + * + * @author Philippe Charles + */ +public enum DataFactories implements DataFactory { + + SDMX20 { + @Override + public FreqParser getFreqParser(DataStructure dsd) { + return FreqParser.sdmx20(); + } + + @Override + public SafeParser getPeriodParser(Frequency freq) { + return FreqUtil.onStandardFreq(freq); + } + + @Override + public SafeParser getValueParser() { + return SafeParser.onStandardDouble(); + } + }, + SDMX21 { + @Override + public FreqParser getFreqParser(DataStructure dsd) { + return FreqParser.sdmx21(dsd); + } + + @Override + public SafeParser getPeriodParser(Frequency freq) { + return FreqUtil.onStandardFreq(freq); + } + + @Override + public SafeParser getValueParser() { + return SafeParser.onStandardDouble(); + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java similarity index 94% rename from sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java rename to sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java index fc5d86c7..cc7f4316 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/FreqParsers.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java @@ -14,11 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.util; +package internal.parser; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.FreqUtil; +import be.nbb.sdmx.facade.parser.FreqUtil; import java.util.function.Function; /** diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java index 80489ade..a484ad7d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java @@ -16,14 +16,10 @@ */ package internal.util; -import be.nbb.sdmx.facade.Frequency; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.Collections; -import java.util.EnumMap; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import be.nbb.sdmx.facade.util.SafeParser; @@ -35,27 +31,6 @@ @lombok.experimental.UtilityClass public class SafeParsers { - public final Map> STANDARD_PARSERS = initStandardParsers(); - - private Map> initStandardParsers() { - SafeParser yearMonth = SafeParser.onDatePattern("yyyy-MM"); - SafeParser yearMonthDay = SafeParser.onDatePattern("yyyy-MM-dd"); - - Map> result = new EnumMap<>(Frequency.class); - result.put(Frequency.ANNUAL, SafeParser.onDatePattern("yyyy").or(SafeParser.onDatePattern("yyyy'-01'")).or(SafeParser.onDatePattern("yyyy'-A1'"))); - result.put(Frequency.HALF_YEARLY, SafeParser.onYearFreqPos("S", 2).or(yearMonth)); - result.put(Frequency.QUARTERLY, SafeParser.onYearFreqPos("Q", 4).or(yearMonth)); - result.put(Frequency.MONTHLY, SafeParser.onYearFreqPos("M", 12).or(yearMonth)); - result.put(Frequency.WEEKLY, yearMonthDay); - result.put(Frequency.DAILY, yearMonthDay); - // FIXME: needs other pattern for time - result.put(Frequency.HOURLY, yearMonthDay); - result.put(Frequency.DAILY_BUSINESS, yearMonthDay); - result.put(Frequency.MINUTELY, yearMonthDay); - result.put(Frequency.UNDEFINED, yearMonth); - return Collections.unmodifiableMap(result); - } - public static final class Fallback implements SafeParser { private final SafeParser first; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java deleted file mode 100644 index 047c9ac6..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/StandardDataFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.util; - -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.util.DataFactory; -import be.nbb.sdmx.facade.util.FreqParser; -import be.nbb.sdmx.facade.util.ObsParser; - -/** - * - * @author Philippe Charles - */ -public enum StandardDataFactory implements DataFactory { - - SDMX20 { - @Override - public FreqParser getFreqParser(DataStructure o) { - return FreqParser.sdmx20(); - } - - @Override - public ObsParser getObsParser(DataStructure o) { - return ObsParser.standard(); - } - }, SDMX21 { - @Override - public FreqParser getFreqParser(DataStructure o) { - return FreqParser.sdmx21(o); - } - - @Override - public ObsParser getObsParser(DataStructure o) { - return ObsParser.standard(); - } - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/FrequencyUtilTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java similarity index 93% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/FrequencyUtilTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java index b2248b56..90c121bf 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/FrequencyUtilTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java @@ -14,18 +14,20 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; +import be.nbb.sdmx.facade.parser.FreqUtil; import be.nbb.sdmx.facade.Frequency; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; /** * * @author Philippe Charles */ -public class FrequencyUtilTest { +public class FreqUtilTest { @Test @SuppressWarnings("null") diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/ObsParserTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/ObsParserTest.java similarity index 99% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/ObsParserTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/ObsParserTest.java index 842c590e..86419829 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/ObsParserTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/ObsParserTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.sdmx.facade.parser; import be.nbb.sdmx.facade.Frequency; import static org.assertj.core.api.Assertions.assertThat; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index 84e5134d..b5e08d87 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -22,11 +22,11 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; -import be.nbb.sdmx.facade.util.ObsParser; +import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import be.nbb.sdmx.facade.parser.ObsParser; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.parser.FreqParser; import java.io.InputStream; import javax.xml.stream.XMLInputFactory; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 12077a2f..0bcad1bb 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -22,11 +22,11 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; -import be.nbb.sdmx.facade.util.ObsParser; +import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import be.nbb.sdmx.facade.parser.ObsParser; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import be.nbb.sdmx.facade.util.FreqParser; +import be.nbb.sdmx.facade.parser.FreqParser; import java.io.InputStream; import javax.xml.stream.XMLInputFactory; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java index 5e1156c7..0d9838f6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.util.NoOpCursor; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.util.UnexpectedIOException; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index 880e95ba..a8a7175d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; import java.io.IOException; import java.time.LocalDateTime; @@ -27,8 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import be.nbb.sdmx.facade.util.FreqUtil; -import static be.nbb.sdmx.facade.util.FreqUtil.TIME_FORMAT_CONCEPT; +import be.nbb.sdmx.facade.parser.FreqUtil; +import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; /** * diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 8bf51a87..784c5154 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -17,10 +17,12 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxFix; import be.nbb.sdmx.facade.util.SdmxMediaType; @@ -41,7 +43,7 @@ import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; import internal.org.springframework.util.xml.XMLEventStreamReader; -import internal.util.drivers.InseeDataFactory; +import internal.util.drivers.InseeDialect; import it.bancaditalia.oss.sdmx.api.Codelist; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.Dimension; @@ -69,12 +71,12 @@ public Collection getDefaultEntryPoints() { private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { @SdmxFix(id = "INSEE#2", cause = "Does not follow sdmx standard codes") - private final InseeDataFactory dataFactory; + private final SdmxDialect dialect; private InseeClient(URI endpoint, LanguagePriorityList langs) { super("", endpoint, false, false, true); this.languages = Util.fromLanguages(langs); - this.dataFactory = new InseeDataFactory(); + this.dialect = new InseeDialect(); } @Override @@ -114,7 +116,9 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd), dataFactory).parse(new XMLEventStreamReader(r), () -> {})) { + DataStructure tmp = Util.toStructure(dsd); + try (DataCursor cursor = SdmxXmlStreams.compactData21(tmp, dialect).parse(new XMLEventStreamReader(r), () -> { + })) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java similarity index 80% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java index d951ded5..c75c1bc6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDataFactory.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java @@ -22,28 +22,44 @@ import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; import static be.nbb.sdmx.facade.Frequency.MONTHLY; import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import be.nbb.sdmx.facade.util.FreqParser; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.FreqParser; import be.nbb.sdmx.facade.util.SafeParser; -import be.nbb.sdmx.facade.util.DataFactory; -import be.nbb.sdmx.facade.util.FreqUtil; +import be.nbb.sdmx.facade.parser.FreqUtil; +import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import java.time.LocalDateTime; +import org.openide.util.lookup.ServiceProvider; /** * https://www.insee.fr/fr/information/2862759 * * @author Philippe Charles */ -public final class InseeDataFactory implements DataFactory { +@ServiceProvider(service = SdmxDialect.class) +public final class InseeDialect implements SdmxDialect { + + @Override + public String getName() { + return "INSEE2017"; + } + + @Override + public String getDescription() { + return getName(); + } @Override public FreqParser getFreqParser(DataStructure dsd) { - return FreqParser.of(FreqUtil.extractorByIndex(dsd), InseeDataFactory::parseInseeFreq); + return FreqParser.of(FreqUtil.extractorByIndex(dsd), InseeDialect::parseInseeFreq); + } + + @Override + public SafeParser getPeriodParser(Frequency freq) { + return onInseeTimePeriod(freq); } @Override - public ObsParser getObsParser(DataStructure dsd) { - return new ObsParser(InseeDataFactory::onInseeTimePeriod, SafeParser.onStandardDouble()); + public SafeParser getValueParser() { + return SafeParser.onStandardDouble(); } private static Frequency parseInseeFreq(String code) { diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java index 81c8ad4d..2311ce26 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.xml.stream.Stax; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index c7dae274..fd604409 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -21,7 +21,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.Obs; -import be.nbb.sdmx.facade.util.ObsParser; +import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.util.SeriesSupport; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; From 6082c16902821dcbe6e9cd085ba00df4f2917bbd Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 20 Nov 2017 16:36:09 +0100 Subject: [PATCH 46/68] Added support of dialects in SdmxFileManager. --- .../nbb/demetra/sdmx/file/SdmxFileBean.java | 1 + .../nbb/demetra/sdmx/file/SdmxFileParam.java | 4 +++ .../demetra/sdmx/file/SdmxFileProvider.java | 2 +- .../java/internal/sdmx/SdmxCubeItems.java | 15 +++++++---- .../sdmx/file/SdmxFileProviderBuddy.java | 16 ++++++++--- .../internal/sdmx/SdmxAutoCompletion.java | 24 +++++++++++++++++ .../nbb/sdmx/facade/file/SdmxFileManager.java | 27 ++++++++++++++++--- .../be/nbb/sdmx/facade/file/SdmxFileSet.java | 6 ++++- .../file/CachedFileSdmxConnection.java | 6 +++-- .../internal/file/FileSdmxConnection.java | 14 ++++++---- .../main/java/internal/file/SdmxFileUtil.java | 13 ++++++++- .../sdmx/facade/file/SdmxFileManagerTest.java | 4 +-- .../nbb/sdmx/facade/file/SdmxFileSetTest.java | 23 +++------------- .../internal/file/FileSdmxConnectionTest.java | 13 ++++----- .../java/internal/file/SdmxFileUtilTest.java | 22 +++++++++++---- 15 files changed, 137 insertions(+), 53 deletions(-) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileBean.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileBean.java index 678ecbd8..1421db8d 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileBean.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileBean.java @@ -30,6 +30,7 @@ public final class SdmxFileBean implements IFileBean { private File file; private File structureFile; + private String dialect; private List dimensions; private String labelAttribute; } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileParam.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileParam.java index 62108bce..c3eb24b1 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileParam.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileParam.java @@ -47,6 +47,7 @@ final class V1 implements SdmxFileParam { private final IParam file = Params.onFile(new File(""), "f"); private final IParam structureFile = Params.onFile(new File(""), "s"); + private final IParam dialect = Params.onString("", "j"); private final IParam> dimensionIds = onStringList(ImmutableList.of(), "d", dimensionSplitter, dimensionJoiner); private final IParam labelAttribute = Params.onString("", "l"); @@ -60,6 +61,7 @@ public SdmxFileBean defaultValue() { SdmxFileBean result = new SdmxFileBean(); result.setFile(file.defaultValue()); result.setStructureFile(structureFile.defaultValue()); + result.setDialect(dialect.defaultValue()); result.setDimensions(dimensionIds.defaultValue()); result.setLabelAttribute(labelAttribute.defaultValue()); return result; @@ -70,6 +72,7 @@ public SdmxFileBean get(DataSource dataSource) { SdmxFileBean result = new SdmxFileBean(); result.setFile(file.get(dataSource)); result.setStructureFile(structureFile.get(dataSource)); + result.setDialect(dialect.get(dataSource)); result.setDimensions(dimensionIds.get(dataSource)); result.setLabelAttribute(labelAttribute.get(dataSource)); return result; @@ -79,6 +82,7 @@ public SdmxFileBean get(DataSource dataSource) { public void set(IConfig.Builder builder, SdmxFileBean value) { file.set(builder, value.getFile()); structureFile.set(builder, value.getStructureFile()); + dialect.set(builder, value.getDialect()); dimensionIds.set(builder, value.getDimensions()); labelAttribute.set(builder, value.getLabelAttribute()); } diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 4fb01704..1b50f23f 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -87,7 +87,7 @@ public SdmxFileProvider() { Cache cache = GuavaCaches.softValuesCache(); SdmxFileParam sdmxParam = new SdmxFileParam.V1(); - this.properties = SdmxPropertiesSupport.of(SdmxFileManager::of, cache::invalidateAll, () -> LanguagePriorityList.ANY, cache::invalidateAll); + this.properties = SdmxPropertiesSupport.of(SdmxFileManager::ofServiceLoader, cache::invalidateAll, () -> LanguagePriorityList.ANY, cache::invalidateAll); this.mutableListSupport = HasDataSourceMutableList.of(NAME, logger, cache::invalidate); this.monikerSupport = HasDataMoniker.usingUri(NAME); this.beanSupport = HasDataSourceBean.of(NAME, sdmxParam, sdmxParam.getVersion()); diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index c5d3c679..fb7fd16a 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -77,10 +77,15 @@ public static Optional tryResolveFileSet(HasFilePaths paths, SdmxFi } public static SdmxFileSet resolveFileSet(HasFilePaths paths, SdmxFileBean bean) throws FileNotFoundException { - return resolveFileSet(paths, bean.getFile(), bean.getStructureFile()); - } - - public static SdmxFileSet resolveFileSet(HasFilePaths paths, File data, File structure) throws FileNotFoundException { - return SdmxFileSet.of(paths.resolveFilePath(data), structure.toString().isEmpty() ? null : paths.resolveFilePath(structure)); + SdmxFileSet.Builder result = SdmxFileSet.builder().data(bean.getFile()); + File structure = bean.getStructureFile(); + if (structure != null && !structure.toString().isEmpty()) { + result.structure(paths.resolveFilePath(structure)); + } + String dialect = bean.getDialect(); + if (dialect != null && !dialect.isEmpty()) { + result.dialect(dialect); + } + return result.build(); } } diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java index d7ffc3e2..d6fe0ea7 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java @@ -128,7 +128,7 @@ public Config editConfig(Config config) throws IllegalArgumentException { // private static SdmxFileManager createManager() { - SdmxFileManager result = SdmxFileManager.of(); + SdmxFileManager result = SdmxFileManager.ofServiceLoader(); result.setCache(GuavaCaches.softValuesCacheAsMap()); return result; } @@ -188,7 +188,7 @@ private Sheet createSheet(SdmxFileBean bean, IFileLoader loader, SdmxConnectionS } @NbBundle.Messages({ - "bean.file.display=Sdmx data file", + "bean.file.display=Data file", "bean.file.description=The path to the sdmx data file.",}) private NodePropertySetBuilder withSource(NodePropertySetBuilder b, SdmxFileBean bean, IFileLoader loader) { b.withFile() @@ -203,8 +203,10 @@ private NodePropertySetBuilder withSource(NodePropertySetBuilder b, SdmxFileBean } @NbBundle.Messages({ - "bean.structureFile.display=Sdmx structure file", + "bean.structureFile.display=Structure file", "bean.structureFile.description=The path to the sdmx structure file.", + "bean.dialect.display=Dialect", + "bean.dialect.description=The name of the dialect used to parse the sdmx data file.", "bean.dimensions.display=Dataflow dimensions", "bean.dimensions.description=An optional comma-separated list of dimensions that defines the order used to hierarchise time series.", "bean.labelAttribute.display=Series label attribute", @@ -220,6 +222,14 @@ private NodePropertySetBuilder withOptions(NodePropertySetBuilder b, SdmxFileBea .directories(false) .add(); + b.withAutoCompletion() + .select(bean, "dialect") + .source(SdmxAutoCompletion.onDialects()) + .cellRenderer(SdmxAutoCompletion.getDialectRenderer()) + .display(Bundle.bean_dialect_display()) + .description(Bundle.bean_dialect_description()) + .add(); + Supplier toSource = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileUtil::toXml).orElse(""); Supplier toFlow = () -> SdmxCubeItems.tryResolveFileSet(loader, bean).map(SdmxFileSet::asDataflowRef).map(Object::toString).orElse(""); diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java index cd7ce915..efcf0be1 100644 --- a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.web.SdmxWebManager; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.UnexpectedIOException; @@ -36,10 +37,12 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.ServiceLoader; import java.util.concurrent.ConcurrentMap; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.swing.ListCellRenderer; /** @@ -49,6 +52,27 @@ @lombok.experimental.UtilityClass public class SdmxAutoCompletion { + public AutoCompletionSource onDialects() { + return ExtAutoCompletionSource + .builder(o -> StreamSupport.stream(ServiceLoader.load(SdmxDialect.class).spliterator(), false).collect(Collectors.toList())) + .behavior(AutoCompletionSource.Behavior.SYNC) + .postProcessor(SdmxAutoCompletion::filterAndSortDialects) + .valueToString(SdmxDialect::getName) + .build(); + } + + private List filterAndSortDialects(List allValues, String term) { + Predicate filter = ExtAutoCompletionSource.basicFilter(term); + return allValues.stream() + .filter(o -> filter.test(o.getDescription()) || filter.test(o.getName())) + .sorted(Comparator.comparing(SdmxDialect::getDescription)) + .collect(Collectors.toList()); + } + + public ListCellRenderer getDialectRenderer() { + return CustomListCellRenderer.of(SdmxDialect::getName, SdmxDialect::getDescription); + } + public AutoCompletionSource onEntryPoints(SdmxWebManager manager) { return ExtAutoCompletionSource .builder(o -> manager.getEntryPoints()) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 13dff468..fdea2daf 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -20,11 +20,17 @@ import internal.file.SdmxDecoder; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; +import be.nbb.sdmx.facade.parser.DataFactory; +import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.xml.stream.Stax; import internal.file.SdmxFileUtil; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; @@ -40,14 +46,22 @@ public final class SdmxFileManager implements SdmxConnectionSupplier, HasCache { @Nonnull - public static SdmxFileManager of() { + public static SdmxFileManager ofServiceLoader() { + List dialects = new ArrayList<>(); + ServiceLoader.load(SdmxDialect.class).forEach(dialects::add); XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); - return new SdmxFileManager(factoryWithoutNamespace, new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), new AtomicReference<>(new ConcurrentHashMap())); + return new SdmxFileManager( + factoryWithoutNamespace, + new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), + new AtomicReference<>(new ConcurrentHashMap()), + dialects + ); } private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final AtomicReference cache; + private final List dialects; @Override public SdmxFileConnection getConnection(String name) throws IOException { @@ -74,7 +88,7 @@ public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files) throws IOExc @Nonnull public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { - return new CachedFileSdmxConnection(files, languages, factoryWithoutNamespace, decoder, cache.get()); + return new CachedFileSdmxConnection(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), cache.get()); } @Override @@ -86,4 +100,11 @@ public ConcurrentMap getCache() { public void setCache(ConcurrentMap cache) { this.cache.set(cache != null ? cache : new ConcurrentHashMap()); } + + private Optional getDataFactory(SdmxFileSet files) { + return dialects.stream() + .filter(o -> o.getName().equals(files.getDialect())) + .map(DataFactory.class::cast) + .findFirst(); + } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java index 897a2a19..0f419a5a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java @@ -25,7 +25,8 @@ * * @author Philippe Charles */ -@lombok.Value(staticConstructor = "of") +@lombok.Value +@lombok.Builder(builderClassName = "Builder") public class SdmxFileSet { @lombok.NonNull @@ -34,6 +35,9 @@ public class SdmxFileSet { @Nullable File structure; + @Nullable + String dialect; + public boolean hasStructure() { return structure != null; } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java index 179c2b68..cc21774c 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java @@ -22,12 +22,14 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.util.TtlCache; import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; import java.time.Clock; import java.util.List; +import java.util.Optional; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import javax.xml.stream.XMLInputFactory; @@ -46,8 +48,8 @@ public final class CachedFileSdmxConnection extends FileSdmxConnection { private final TypedId decodeKey; private final TypedId> loadDataKey; - public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, ConcurrentMap cache) { - super(files, languages, factoryWithoutNamespace, decoder); + public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory, ConcurrentMap cache) { + super(files, languages, factoryWithoutNamespace, decoder, dataFactory); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); String base = SdmxFileUtil.toXml(files) + languages.toString(); this.decodeKey = TypedId.of("decode://" + base); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java index 602ac8d0..d83dc896 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java @@ -28,6 +28,7 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileConnection; import be.nbb.sdmx.facade.file.SdmxFileSet; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; @@ -38,6 +39,7 @@ import java.util.stream.Stream; import javax.xml.stream.XMLInputFactory; import be.nbb.sdmx.facade.xml.stream.Stax; +import java.util.Optional; /** * @@ -51,14 +53,16 @@ class FileSdmxConnection implements SdmxFileConnection { private final LanguagePriorityList languages; private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; + private final Optional dataFactory; private final Dataflow dataflow; private boolean closed; - FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder) { + FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory) { this.files = files; this.languages = languages; this.factoryWithoutNamespace = factoryWithoutNamespace; this.decoder = decoder; + this.dataFactory = dataFactory; this.dataflow = Dataflow.of(files.asDataflowRef(), EMPTY, SdmxFileUtil.asFlowLabel(files)); this.closed = false; } @@ -156,13 +160,13 @@ protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key k private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { switch (o) { case GENERIC20: - return SdmxXmlStreams.genericData20(dsd); + return SdmxXmlStreams.genericData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); case COMPACT20: - return SdmxXmlStreams.compactData20(dsd); + return SdmxXmlStreams.compactData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); case GENERIC21: - return SdmxXmlStreams.genericData21(dsd); + return SdmxXmlStreams.genericData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); case COMPACT21: - return SdmxXmlStreams.compactData21(dsd); + return SdmxXmlStreams.compactData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); default: throw new IOException("Don't known how to handle type '" + o + "'"); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index d214fcdc..06d1e223 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -50,6 +50,10 @@ public String toXml(@Nonnull SdmxFileSet files) { if (files.hasStructure()) { xml.writeAttribute(STRUCT_ATTR, files.getStructure().toString()); } + String dialect = files.getDialect(); + if (dialect != null && !dialect.isEmpty()) { + xml.writeAttribute(DIALECT_ATTR, dialect); + } xml.writeEndDocument(); xml.close(); } catch (XMLStreamException ex) { @@ -62,12 +66,14 @@ public String toXml(@Nonnull SdmxFileSet files) { public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentException { String data = null; String structure = null; + String dialect = null; try { XMLStreamReader xml = Stax.getInputFactoryWithoutNamespace().createXMLStreamReader(new StringReader(input)); while (xml.hasNext()) { if (xml.next() == XMLStreamReader.START_ELEMENT && xml.getLocalName().equals(ROOT_TAG)) { data = xml.getAttributeValue(null, DATA_ATTR); structure = xml.getAttributeValue(null, STRUCT_ATTR); + dialect = xml.getAttributeValue(null, DIALECT_ATTR); } } xml.close(); @@ -77,11 +83,16 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE if (data == null || data.isEmpty()) { throw new IllegalArgumentException("Cannot parse SdmxFile from '" + input + "'"); } - return SdmxFileSet.of(new File(data), structure != null && !structure.isEmpty() ? new File(structure) : null); + return SdmxFileSet.builder() + .data(new File(data)) + .structure(structure != null && !structure.isEmpty() ? new File(structure) : null) + .dialect(dialect) + .build(); } private static final String ROOT_TAG = "file"; private static final String DATA_ATTR = "data"; private static final String STRUCT_ATTR = "structure"; + private static final String DIALECT_ATTR = "dialect"; private static final XMLOutputFactory OUTPUT = XMLOutputFactory.newInstance(); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index bf9b59e2..40f7f0db 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -39,8 +39,8 @@ public void testCompliance() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFileSet files = SdmxFileSet.of(compact21, null); + SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - ConnectionSupplierAssert.assertCompliance(SdmxFileManager.of(), SdmxFileUtil.toXml(files), "ko"); + ConnectionSupplierAssert.assertCompliance(SdmxFileManager.ofServiceLoader(), SdmxFileUtil.toXml(files), "ko"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java index fd497b40..820b4519 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileSetTest.java @@ -17,8 +17,7 @@ package be.nbb.sdmx.facade.file; import java.io.File; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -27,29 +26,15 @@ */ public class SdmxFileSetTest { - @Test - @SuppressWarnings("null") - public void testFactory() { - assertThatThrownBy(() -> SdmxFileSet.of(null, null)).isInstanceOf(NullPointerException.class); - - assertThat(SdmxFileSet.of(data, null)) - .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", null); - - assertThat(SdmxFileSet.of(data, structure)) - .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", structure); - } - @Test public void testAsDataflowRef() { - assertThat(SdmxFileSet.of(data, structure).asDataflowRef().toString()) + assertThat(SdmxFileSet.builder().data(data).structure(structure).build().asDataflowRef().toString()) .isEqualTo("all,data&struct,latest"); - assertThat(SdmxFileSet.of(data, new File("")).asDataflowRef().toString()) + assertThat(SdmxFileSet.builder().data(data).structure(new File("")).build().asDataflowRef().toString()) .isEqualTo("all,data,latest"); - assertThat(SdmxFileSet.of(data, null).asDataflowRef().toString()) + assertThat(SdmxFileSet.builder().data(data).build().asDataflowRef().toString()) .isEqualTo("all,data,latest"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java index 23bee8f2..20e65e66 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java @@ -18,9 +18,9 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.DataQuery; +import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; @@ -28,6 +28,7 @@ import be.nbb.sdmx.facade.xml.stream.Stax; import java.io.File; import java.io.IOException; +import java.util.Optional; import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -46,9 +47,9 @@ public void testFile() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFileSet files = SdmxFileSet.of(compact21, null); + SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder); + FileSdmxConnection conn = new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); assertThat(conn.getFlow()).isEqualTo(conn.getFlow(files.asDataflowRef())); @@ -63,9 +64,9 @@ public void testCompactData21() throws IOException { File compact21 = temp.newFile(); SdmxSource.OTHER_COMPACT21.copyTo(compact21); - SdmxFileSet files = SdmxFileSet.of(compact21, null); + SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - FileSdmxConnection conn = new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder); + FileSdmxConnection conn = new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); assertThat(conn.getFlows()).hasSize(1); assertThat(conn.getStructure(files.asDataflowRef()).getDimensions()).hasSize(7); @@ -93,7 +94,7 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, LanguagePriorityList.ANY, factoryWithoutNamespace, decoder), files.asDataflowRef()); + ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()), files.asDataflowRef()); } @Rule diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java index 34b623e2..7dce2cc1 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java @@ -30,10 +30,13 @@ public class SdmxFileUtilTest { @Test public void testToXml() { - assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, structure))) + assertThat(SdmxFileUtil.toXml(SdmxFileSet.builder().data(data).structure(structure).dialect("hello").build())) + .isEqualTo(""); + + assertThat(SdmxFileUtil.toXml(SdmxFileSet.builder().data(data).structure(structure).build())) .isEqualTo(""); - assertThat(SdmxFileUtil.toXml(SdmxFileSet.of(data, null))) + assertThat(SdmxFileUtil.toXml(SdmxFileSet.builder().data(data).build())) .isEqualTo(""); } @@ -47,17 +50,26 @@ public void testFromXml() { assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", null); + .hasFieldOrPropertyWithValue("structure", null) + .hasFieldOrPropertyWithValue("dialect", null); assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", null); + .hasFieldOrPropertyWithValue("structure", null) + .hasFieldOrPropertyWithValue("dialect", null); assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) - .hasFieldOrPropertyWithValue("structure", structure); + .hasFieldOrPropertyWithValue("structure", structure) + .hasFieldOrPropertyWithValue("dialect", null); + + assertThat(SdmxFileUtil.fromXml("")) + .hasFieldOrPropertyWithValue("data", data) + .hasFieldOrPropertyWithValue("structure", structure) + .hasFieldOrPropertyWithValue("dialect", dialect); } private final File data = new File("a.xml"); private final File structure = new File("b.xml"); + private final String dialect = "hello"; } From 665089eeb702a6539c32c83dc3d2b8e2d97ed5a1 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 21 Nov 2017 11:52:27 +0100 Subject: [PATCH 47/68] Fixed INSEE dimensions & codes. --- .../connectors/GenericSDMXClientResource.java | 7 +--- .../main/java/internal/connectors/Util.java | 6 ++++ .../connectors/drivers/InseeDriver.java | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java index 0d9838f6..0c92715b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java @@ -26,7 +26,6 @@ import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import it.bancaditalia.oss.sdmx.client.custom.DotStat; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.exceptions.SdmxResponseException; import java.io.IOException; import java.util.Map; import java.util.logging.Level; @@ -112,7 +111,7 @@ public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) ? ((HasDataCursor) client).getDataCursor(dataflow, dfs, key, serieskeysonly) : new PortableTimeSeriesCursor(client.getTimeSeries(dataflow, dfs, key.toString(), null, null, serieskeysonly, null, false), ObsParser.standard()); } catch (SdmxException ex) { - if (isNoResultMatchingQuery(ex)) { + if (Util.isNoResultMatchingQuery(ex)) { return NoOpCursor.noOp(); } throw expected(ex, "Failed to get data from dataset '%s' with key '%s'", flowRef, key); @@ -153,8 +152,4 @@ private static IOException unexpectedNull(String format, Object... args) { log.log(Level.WARNING, msg); return new UnexpectedIOException(new NullPointerException(msg)); } - - private static boolean isNoResultMatchingQuery(SdmxException ex) { - return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == 100; - } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 77cc89bd..26af185f 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -24,6 +24,8 @@ import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.api.Dimension; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import it.bancaditalia.oss.sdmx.exceptions.SdmxResponseException; /** * @@ -66,6 +68,10 @@ public it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.s return it.bancaditalia.oss.sdmx.util.LanguagePriorityList.parse(l.toString()); } + public boolean isNoResultMatchingQuery(SdmxException ex) { + return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == 100; + } + public static final BoolProperty SUPPORTS_COMPRESSION = new BoolProperty("supportsCompression"); public static final BoolProperty NEEDS_CREDENTIALS = new BoolProperty("needsCredentials"); public static final BoolProperty NEEDS_URL_ENCODING = new BoolProperty("needsURLEncoding"); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 784c5154..69b06b97 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -49,11 +49,13 @@ import it.bancaditalia.oss.sdmx.api.Dimension; import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; +import java.util.logging.Level; /** * * @author Philippe Charles */ +@lombok.extern.java.Log @ServiceProvider(service = SdmxWebDriver.class) public final class InseeDriver implements SdmxWebDriver, HasCache { @@ -82,6 +84,7 @@ private InseeClient(URI endpoint, LanguagePriorityList langs) { @Override public DataFlowStructure getDataFlowStructure(DSDIdentifier dsd, boolean full) throws SdmxException { DataFlowStructure result = super.getDataFlowStructure(dsd, full); + fixIds(result); fixMissingCodes(result); return result; } @@ -97,13 +100,38 @@ public boolean isSeriesKeysOnlySupported() { return true; } + @SdmxFix(id = "INSEE#4", cause = "Some dimension/code ids are invalid") + private void fixIds(DataFlowStructure dsd) { + for (Dimension d : dsd.getDimensions()) { + if (d.getId().endsWith("6")) { + d.setId(getValidId(d.getId())); +// d.getCodeList().setId(getValidId(d.getCodeList().getId())); + } + } + } + + private String getValidId(String id) { + return id.substring(0, id.length() - 1); + } + @SdmxFix(id = "INSEE#3", cause = "Some codes are missing in dsd even when requested with 'references=children'") private void fixMissingCodes(DataFlowStructure dsd) throws SdmxException { for (Dimension d : dsd.getDimensions()) { - Codelist freq = d.getCodeList(); - if (freq.getCodes().isEmpty()) { - freq.setCodes(super.getCodes(freq.getId(), freq.getAgency(), freq.getVersion())); + Codelist codelist = d.getCodeList(); + if (codelist.getCodes().isEmpty()) { + loadMissingCodes(codelist); + } + } + } + + private void loadMissingCodes(Codelist codelist) throws SdmxException { + try { + codelist.setCodes(super.getCodes(codelist.getId(), codelist.getAgency(), codelist.getVersion())); + } catch (SdmxException ex) { + if (!Util.isNoResultMatchingQuery(ex)) { + throw ex; } + log.log(Level.WARNING, "Cannot retrieve codes for ''{0}''", codelist.getFullIdentifier()); } } From 38243f9e7e47286de57cf79802cf9130ba34aaa2 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 21 Nov 2017 12:55:38 +0100 Subject: [PATCH 48/68] Fixed annoying logs. --- .../sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java | 4 ++-- .../sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 063ba517..61b01658 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -32,12 +32,12 @@ import be.nbb.sdmx.facade.parser.FreqParser; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; -import java.util.logging.Logger; /** * * @author Philippe Charles */ +@lombok.extern.java.Log final class XMLStreamCompactDataCursor implements DataCursor { private static final String DATASET_TAG = "DataSet"; @@ -58,7 +58,7 @@ final class XMLStreamCompactDataCursor implements DataCursor { XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, String timeDimensionId, String primaryMeasureId) { if (!Stax.isNotNamespaceAware(reader)) { - Logger.getLogger(getClass().getName()).warning("Using XMLStreamReader with namespace awareness"); + log.fine("Using XMLStreamReader with namespace awareness"); } this.reader = reader; this.onClose = onClose; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 666503db..cc178979 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -34,13 +34,13 @@ import java.io.Closeable; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.util.logging.Logger; import javax.annotation.Nonnull; /** * * @author Philippe Charles */ +@lombok.extern.java.Log final class XMLStreamGenericDataCursor implements DataCursor { static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { @@ -73,7 +73,7 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClo private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { if (!Stax.isNotNamespaceAware(reader)) { - Logger.getLogger(getClass().getName()).warning("Using XMLStreamReader with namespace awareness"); + log.fine("Using XMLStreamReader with namespace awareness"); } this.reader = reader; this.onClose = onClose; From a9ca07f28306b89cd16c4e0d396cc03484aa11df Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 21 Nov 2017 14:28:21 +0100 Subject: [PATCH 49/68] Code cleanup. --- .../java/internal/sdmx/SdmxCubeItems.java | 2 +- .../sdmx/file/SdmxFileProviderTest.java | 2 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 41 ++-- .../be/nbb/sdmx/facade/file/SdmxFileSet.java | 4 - ...dmxConnection.java => CachedResource.java} | 8 +- .../internal/file/FileSdmxConnection.java | 180 ------------------ .../main/java/internal/file/SdmxDecoder.java | 6 +- .../internal/file/SdmxDecoderResource.java | 71 +++++++ .../internal/file/SdmxFileConnectionImpl.java | 138 ++++++++++++++ .../main/java/internal/file/SdmxFileUtil.java | 19 +- .../{ => xml}/CustomDataStructureBuilder.java | 2 +- .../file/{ => xml}/DataStructureDecoder.java | 4 +- .../file/{ => xml}/DataTypeProbe.java | 3 +- .../StaxSdmxDecoder.java} | 15 +- .../sdmx/facade/file/SdmxFileManagerTest.java | 15 ++ ...t.java => SdmxFileConnectionImplTest.java} | 17 +- .../{ => xml}/DataStructureDecoderTest.java | 4 +- .../file/{ => xml}/DataTypeProbeTest.java | 3 +- .../nbb/sdmx/facade/parser/DataFactory.java | 8 +- .../be/nbb/sdmx/facade/parser/FreqParser.java | 67 ------- .../parser/{FreqUtil.java => Freqs.java} | 67 +++++-- .../be/nbb/sdmx/facade/parser/ObsParser.java | 12 +- .../nbb/sdmx/facade/util/Chars.java} | 71 ++++++- .../be/nbb/sdmx/facade/util/SafeParser.java | 78 -------- .../stream/XMLStreamCompactDataCursor.java | 6 +- .../stream/XMLStreamGenericDataCursor.java | 10 +- .../java/internal/parser/DataFactories.java | 29 ++- .../java/internal/parser/FreqParsers.java | 35 ---- .../nbb/sdmx/facade/parser/FreqUtilTest.java | 34 ++-- .../XMLStreamCompactDataCursorTest.java | 12 +- .../XMLStreamGenericDataCursorTest.java | 16 +- .../connectors/PortableTimeSeriesCursor.java | 8 +- .../internal/util/drivers/InseeDialect.java | 27 ++- 33 files changed, 502 insertions(+), 512 deletions(-) rename sdmx-facade/sdmx-facade-file/src/main/java/internal/file/{CachedFileSdmxConnection.java => CachedResource.java} (85%) delete mode 100644 sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java create mode 100644 sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java create mode 100644 sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java rename sdmx-facade/sdmx-facade-file/src/main/java/internal/file/{ => xml}/CustomDataStructureBuilder.java (99%) rename sdmx-facade/sdmx-facade-file/src/main/java/internal/file/{ => xml}/DataStructureDecoder.java (99%) rename sdmx-facade/sdmx-facade-file/src/main/java/internal/file/{ => xml}/DataTypeProbe.java (98%) rename sdmx-facade/sdmx-facade-file/src/main/java/internal/file/{XMLStreamSdmxDecoder.java => xml/StaxSdmxDecoder.java} (88%) rename sdmx-facade/sdmx-facade-file/src/test/java/internal/file/{FileSdmxConnectionTest.java => SdmxFileConnectionImplTest.java} (80%) rename sdmx-facade/sdmx-facade-file/src/test/java/internal/file/{ => xml}/DataStructureDecoderTest.java (97%) rename sdmx-facade/sdmx-facade-file/src/test/java/internal/file/{ => xml}/DataTypeProbeTest.java (97%) delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/{FreqUtil.java => Freqs.java} (66%) rename sdmx-facade/sdmx-facade-util/src/main/java/{internal/util/SafeParsers.java => be/nbb/sdmx/facade/util/Chars.java} (53%) delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index fb7fd16a..fb550850 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -77,7 +77,7 @@ public static Optional tryResolveFileSet(HasFilePaths paths, SdmxFi } public static SdmxFileSet resolveFileSet(HasFilePaths paths, SdmxFileBean bean) throws FileNotFoundException { - SdmxFileSet.Builder result = SdmxFileSet.builder().data(bean.getFile()); + SdmxFileSet.Builder result = SdmxFileSet.builder().data(paths.resolveFilePath(bean.getFile())); File structure = bean.getStructureFile(); if (structure != null && !structure.toString().isEmpty()) { result.structure(paths.resolveFilePath(structure)); diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java index ee8d9eb7..09b781df 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; -import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; import ec.tss.TsCollectionInformation; import ec.tss.TsInformation; import ec.tss.TsInformationType; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index fdea2daf..b7ab7858 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -16,16 +16,19 @@ */ package be.nbb.sdmx.facade.file; -import internal.file.CachedFileSdmxConnection; -import internal.file.SdmxDecoder; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; -import internal.file.XMLStreamSdmxDecoder; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.file.CachedResource; +import internal.file.SdmxDecoder; +import internal.file.SdmxFileConnectionImpl; import internal.file.SdmxFileUtil; +import internal.file.xml.StaxSdmxDecoder; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -52,12 +55,14 @@ public static SdmxFileManager ofServiceLoader() { XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); return new SdmxFileManager( factoryWithoutNamespace, - new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), + new StaxSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), new AtomicReference<>(new ConcurrentHashMap()), dialects ); } + private static final DataStructureRef EMPTY = DataStructureRef.of("", "", ""); + private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final AtomicReference cache; @@ -70,15 +75,7 @@ public SdmxFileConnection getConnection(String name) throws IOException { @Override public SdmxFileConnection getConnection(String name, LanguagePriorityList languages) throws IOException { - SdmxFileSet files; - - try { - files = SdmxFileUtil.fromXml(name); - } catch (IllegalArgumentException ex) { - throw new IOException(ex.getMessage(), ex.getCause()); - } - - return getConnection(files, languages); + return getConnection(getFiles(name), languages); } @Nonnull @@ -88,7 +85,7 @@ public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files) throws IOExc @Nonnull public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException { - return new CachedFileSdmxConnection(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), cache.get()); + return new SdmxFileConnectionImpl(getResource(files, languages), getDataflow(files)); } @Override @@ -101,6 +98,22 @@ public void setCache(ConcurrentMap cache) { this.cache.set(cache != null ? cache : new ConcurrentHashMap()); } + private SdmxFileSet getFiles(String name) throws IOException { + try { + return SdmxFileUtil.fromXml(name); + } catch (IllegalArgumentException ex) { + throw new IOException(ex.getMessage(), ex.getCause()); + } + } + + private SdmxFileConnectionImpl.Resource getResource(SdmxFileSet files, LanguagePriorityList languages) { + return new CachedResource(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), cache.get()); + } + + private Dataflow getDataflow(SdmxFileSet files) { + return Dataflow.of(files.asDataflowRef(), EMPTY, SdmxFileUtil.asFlowLabel(files)); + } + private Optional getDataFactory(SdmxFileSet files) { return dialects.stream() .filter(o -> o.getName().equals(files.getDialect())) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java index 0f419a5a..67a3acfb 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileSet.java @@ -38,10 +38,6 @@ public class SdmxFileSet { @Nullable String dialect; - public boolean hasStructure() { - return structure != null; - } - @Nonnull public DataflowRef asDataflowRef() { return DataflowRef.parse("data" + (structure != null && !structure.toString().isEmpty() ? "&struct" : "")); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java similarity index 85% rename from sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java index cc21774c..4079c94d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedFileSdmxConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java @@ -38,7 +38,7 @@ * * @author Philippe Charles */ -public final class CachedFileSdmxConnection extends FileSdmxConnection { +public final class CachedResource extends SdmxDecoderResource { // TODO: replace ttl with file last modification time private static final long DEFAULT_CACHE_TTL = TimeUnit.MINUTES.toMillis(5); @@ -48,7 +48,7 @@ public final class CachedFileSdmxConnection extends FileSdmxConnection { private final TypedId decodeKey; private final TypedId> loadDataKey; - public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory, ConcurrentMap cache) { + public CachedResource(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory, ConcurrentMap cache) { super(files, languages, factoryWithoutNamespace, decoder, dataFactory); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); String base = SdmxFileUtil.toXml(files) + languages.toString(); @@ -57,7 +57,7 @@ public CachedFileSdmxConnection(SdmxFileSet files, LanguagePriorityList language } @Override - protected SdmxDecoder.Info decode() throws IOException { + public SdmxDecoder.Info decode() throws IOException { SdmxDecoder.Info result = cache.get(decodeKey); if (result == null) { result = super.decode(); @@ -67,7 +67,7 @@ protected SdmxDecoder.Info decode() throws IOException { } @Override - protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + public DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { if (serieskeysonly) { List result = cache.get(loadDataKey); if (result == null) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java deleted file mode 100644 index d83dc896..00000000 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/FileSdmxConnection.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.file; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.DataQueryDetail; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.file.SdmxFileConnection; -import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.sdmx.facade.parser.DataFactory; -import be.nbb.sdmx.facade.util.SeriesSupport; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Stream; -import javax.xml.stream.XMLInputFactory; -import be.nbb.sdmx.facade.xml.stream.Stax; -import java.util.Optional; - -/** - * - * @author Philippe Charles - */ -class FileSdmxConnection implements SdmxFileConnection { - - private static final DataStructureRef EMPTY = DataStructureRef.of("", "", ""); - - private final SdmxFileSet files; - private final LanguagePriorityList languages; - private final XMLInputFactory factoryWithoutNamespace; - private final SdmxDecoder decoder; - private final Optional dataFactory; - private final Dataflow dataflow; - private boolean closed; - - FileSdmxConnection(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory) { - this.files = files; - this.languages = languages; - this.factoryWithoutNamespace = factoryWithoutNamespace; - this.decoder = decoder; - this.dataFactory = dataFactory; - this.dataflow = Dataflow.of(files.asDataflowRef(), EMPTY, SdmxFileUtil.asFlowLabel(files)); - this.closed = false; - } - - @Override - final public DataflowRef getDataflowRef() throws IOException { - checkState(); - return dataflow.getRef(); - } - - @Override - final public Set getFlows() throws IOException { - checkState(); - return Collections.singleton(dataflow); - } - - @Override - final public Dataflow getFlow() throws IOException { - checkState(); - return dataflow; - } - - @Override - final public Dataflow getFlow(DataflowRef flowRef) throws IOException { - checkState(); - Objects.requireNonNull(flowRef); - checkFlowRef(flowRef); - return dataflow; - } - - @Override - final public DataStructure getStructure() throws IOException { - checkState(); - return decode().getDataStructure(); - } - - @Override - final public DataStructure getStructure(DataflowRef flowRef) throws IOException { - checkState(); - Objects.requireNonNull(flowRef); - checkFlowRef(flowRef); - return decode().getDataStructure(); - } - - @Override - final public DataCursor getCursor(DataQuery query) throws IOException { - checkState(); - Objects.requireNonNull(query); - return loadData(decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); - } - - @Override - final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { - checkState(); - Objects.requireNonNull(flowRef); - Objects.requireNonNull(query); - checkFlowRef(flowRef); - return loadData(decode(), flowRef, query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); - } - - @Override - public Stream getStream(DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getCursor(query)); - } - - @Override - public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getCursor(flowRef, query)); - } - - @Override - final public boolean isSeriesKeysOnlySupported() { - return true; - } - - @Override - public void close() throws IOException { - closed = true; - } - - private void checkState() throws IOException { - if (closed) { - throw new IOException("Connection closed"); - } - } - - protected SdmxDecoder.Info decode() throws IOException { - return decoder.decode(files, languages); - } - - protected DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - return getDataSupplier(entry.getDataType(), entry.getDataStructure()).parseFile(factoryWithoutNamespace, files.getData(), StandardCharsets.UTF_8); - } - - private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { - switch (o) { - case GENERIC20: - return SdmxXmlStreams.genericData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); - case COMPACT20: - return SdmxXmlStreams.compactData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); - case GENERIC21: - return SdmxXmlStreams.genericData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); - case COMPACT21: - return SdmxXmlStreams.compactData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); - default: - throw new IOException("Don't known how to handle type '" + o + "'"); - } - } - - private void checkFlowRef(DataflowRef flowRef) throws IOException { - if (!this.dataflow.getRef().contains(flowRef)) { - throw new IOException("Invalid flowref '" + flowRef + "'"); - } - } -} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java index f64008fa..19200860 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoder.java @@ -29,7 +29,7 @@ public interface SdmxDecoder { @Nonnull - Info decode(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList ranges) throws IOException; + Info decode(@Nonnull SdmxFileSet files, @Nonnull LanguagePriorityList languages) throws IOException; enum DataType { @@ -39,7 +39,7 @@ enum DataType { @lombok.Value(staticConstructor = "of") class Info { - DataType dataType; - DataStructure dataStructure; + DataType type; + DataStructure structure; } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java new file mode 100644 index 00000000..0a884c28 --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.file; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.file.SdmxFileSet; +import be.nbb.sdmx.facade.parser.DataFactory; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import be.nbb.sdmx.facade.xml.stream.Stax; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import javax.xml.stream.XMLInputFactory; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor +class SdmxDecoderResource implements SdmxFileConnectionImpl.Resource { + + private final SdmxFileSet files; + private final LanguagePriorityList languages; + private final XMLInputFactory factoryWithoutNamespace; + private final SdmxDecoder decoder; + private final Optional dataFactory; + + @Override + public SdmxDecoder.Info decode() throws IOException { + return decoder.decode(files, languages); + } + + @Override + public DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { + return getDataSupplier(entry.getType(), entry.getStructure()) + .parseFile(factoryWithoutNamespace, files.getData(), StandardCharsets.UTF_8); + } + + private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { + switch (o) { + case GENERIC20: + return SdmxXmlStreams.genericData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); + case COMPACT20: + return SdmxXmlStreams.compactData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); + case GENERIC21: + return SdmxXmlStreams.genericData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); + case COMPACT21: + return SdmxXmlStreams.compactData21(dsd, dataFactory.orElse(DataFactory.sdmx21())); + default: + throw new IOException("Don't known how to handle type '" + o + "'"); + } + } +} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java new file mode 100644 index 00000000..fb76bc13 --- /dev/null +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java @@ -0,0 +1,138 @@ +/* + * Copyright 2015 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.file; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.file.SdmxFileConnection; +import be.nbb.sdmx.facade.util.SeriesSupport; +import java.io.IOException; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; + +/** + * + * @author Philippe Charles + */ +@lombok.RequiredArgsConstructor +public final class SdmxFileConnectionImpl implements SdmxFileConnection { + + public interface Resource { + + SdmxDecoder.Info decode() throws IOException; + + DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException; + } + + private final Resource resource; + private final Dataflow dataflow; + private boolean closed = false; + + @Override + public DataflowRef getDataflowRef() throws IOException { + checkState(); + return dataflow.getRef(); + } + + @Override + public Set getFlows() throws IOException { + checkState(); + return Collections.singleton(dataflow); + } + + @Override + public Dataflow getFlow() throws IOException { + checkState(); + return dataflow; + } + + @Override + final public Dataflow getFlow(DataflowRef flowRef) throws IOException { + checkState(); + checkFlowRef(flowRef); + return dataflow; + } + + @Override + public DataStructure getStructure() throws IOException { + checkState(); + return resource.decode().getStructure(); + } + + @Override + public DataStructure getStructure(DataflowRef flowRef) throws IOException { + checkState(); + checkFlowRef(flowRef); + return resource.decode().getStructure(); + } + + @Override + public DataCursor getCursor(DataQuery query) throws IOException { + checkState(); + Objects.requireNonNull(query); + return resource.loadData(resource.decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); + } + + @Override + final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { + checkState(); + checkFlowRef(flowRef); + Objects.requireNonNull(query); + return resource.loadData(resource.decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); + } + + @Override + public Stream getStream(DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getCursor(query)); + } + + @Override + public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { + return SeriesSupport.asStream(() -> getCursor(flowRef, query)); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return true; + } + + @Override + public void close() throws IOException { + closed = true; + } + + private void checkState() throws IOException { + if (closed) { + throw new IOException("Connection closed"); + } + } + + private void checkFlowRef(DataflowRef flowRef) throws IOException { + Objects.requireNonNull(flowRef); + if (!this.dataflow.getRef().contains(flowRef)) { + throw new IOException("Invalid flowref '" + flowRef + "'"); + } + } +} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 06d1e223..6564071a 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -46,14 +46,19 @@ public String toXml(@Nonnull SdmxFileSet files) { try { XMLStreamWriter xml = OUTPUT.createXMLStreamWriter(result); xml.writeEmptyElement(ROOT_TAG); + xml.writeAttribute(DATA_ATTR, files.getData().toString()); - if (files.hasStructure()) { + + File structure = files.getStructure(); + if (isValidFile(structure)) { xml.writeAttribute(STRUCT_ATTR, files.getStructure().toString()); } + String dialect = files.getDialect(); - if (dialect != null && !dialect.isEmpty()) { + if (!isNullOrEmpty(dialect)) { xml.writeAttribute(DIALECT_ATTR, dialect); } + xml.writeEndDocument(); xml.close(); } catch (XMLStreamException ex) { @@ -85,11 +90,19 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE } return SdmxFileSet.builder() .data(new File(data)) - .structure(structure != null && !structure.isEmpty() ? new File(structure) : null) + .structure(!isNullOrEmpty(structure) ? new File(structure) : null) .dialect(dialect) .build(); } + public boolean isValidFile(File file) { + return file != null && !file.toString().isEmpty(); + } + + private boolean isNullOrEmpty(String o) { + return o == null || o.isEmpty(); + } + private static final String ROOT_TAG = "file"; private static final String DATA_ATTR = "data"; private static final String STRUCT_ATTR = "structure"; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/CustomDataStructureBuilder.java similarity index 99% rename from sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/CustomDataStructureBuilder.java index 1c72d685..f4a67c92 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CustomDataStructureBuilder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/CustomDataStructureBuilder.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java similarity index 99% rename from sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java index 8ef19833..f30e27a2 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java @@ -14,11 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import be.nbb.sdmx.facade.DataStructure; import static internal.file.SdmxDecoder.DataType.*; -import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java similarity index 98% rename from sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java index b9b8224b..e742ecf5 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import static be.nbb.sdmx.facade.xml.Sdmxml.*; import static internal.file.SdmxDecoder.DataType.*; @@ -23,6 +23,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.file.SdmxDecoder; /** * diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java similarity index 88% rename from sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java rename to sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java index 1510abae..1a174f31 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/XMLStreamSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.LanguagePriorityList; @@ -30,23 +30,26 @@ import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.util.List; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.file.SdmxDecoder; +import internal.file.SdmxFileUtil; /** * * @author Philippe Charles */ @lombok.AllArgsConstructor -public final class XMLStreamSdmxDecoder implements SdmxDecoder { +public final class StaxSdmxDecoder implements SdmxDecoder { private final XMLInputFactory factory; private final XMLInputFactory factoryWithoutNamespace; @Override public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { - DataType dataType = probeDataType(files.getData()); - return Info.of(dataType, files.hasStructure() - ? parseStruct(dataType, langs, files.getStructure()) - : decodeStruct(dataType, files.getData())); + DataType type = probeDataType(files.getData()); + File structure = files.getStructure(); + return Info.of(type, SdmxFileUtil.isValidFile(structure) + ? parseStruct(type, langs, structure) + : decodeStruct(type, files.getData())); } private DataType probeDataType(File data) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index 40f7f0db..14316069 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -16,11 +16,13 @@ */ package be.nbb.sdmx.facade.file; +import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; import internal.file.SdmxFileUtil; import java.io.File; import java.io.IOException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -43,4 +45,17 @@ public void testCompliance() throws IOException { ConnectionSupplierAssert.assertCompliance(SdmxFileManager.ofServiceLoader(), SdmxFileUtil.toXml(files), "ko"); } + + @Test + @SuppressWarnings("null") + public void test() { + SdmxFileManager m = SdmxFileManager.ofServiceLoader(); + assertThatThrownBy(() -> m.getConnection((String) null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> m.getConnection((String) null, LanguagePriorityList.ANY)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> m.getConnection(SdmxFileUtil.toXml(files), null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> m.getConnection((SdmxFileSet) null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> m.getConnection(files, null)).isInstanceOf(NullPointerException.class); + } + + private final SdmxFileSet files = SdmxFileSet.builder().data(new File("hello")).build(); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java similarity index 80% rename from sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index 20e65e66..d7b217eb 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/FileSdmxConnectionTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -16,10 +16,14 @@ */ package internal.file; +import internal.file.xml.StaxSdmxDecoder; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileSet; @@ -39,7 +43,7 @@ * * @author Philippe Charles */ -public class FileSdmxConnectionTest { +public class SdmxFileConnectionImplTest { @Test @SuppressWarnings("null") @@ -49,7 +53,8 @@ public void testFile() throws IOException { SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - FileSdmxConnection conn = new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl conn = new SdmxFileConnectionImpl(r, dataflow); assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); assertThat(conn.getFlow()).isEqualTo(conn.getFlow(files.asDataflowRef())); @@ -66,7 +71,8 @@ public void testCompactData21() throws IOException { SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - FileSdmxConnection conn = new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl conn = new SdmxFileConnectionImpl(r, dataflow); assertThat(conn.getFlows()).hasSize(1); assertThat(conn.getStructure(files.asDataflowRef()).getDimensions()).hasSize(7); @@ -94,12 +100,13 @@ public void testCompactData21() throws IOException { assertThat(o.nextSeries()).isFalse(); } - ConnectionAssert.assertCompliance(() -> new FileSdmxConnection(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()), files.asDataflowRef()); + ConnectionAssert.assertCompliance(() -> new SdmxFileConnectionImpl(r, dataflow), files.asDataflowRef()); } @Rule public TemporaryFolder temp = new TemporaryFolder(); private final XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); - private final SdmxDecoder decoder = new XMLStreamSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace); + private final SdmxDecoder decoder = new StaxSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace); + private final Dataflow dataflow = Dataflow.of(DataflowRef.parse("data"), DataStructureRef.parse("xyz"), "label"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index fb2b454c..60ffb49f 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.xml.stream.Stax; import static org.assertj.core.api.Assertions.assertThat; -import static internal.file.CustomDataStructureBuilder.dimension; +import static internal.file.xml.CustomDataStructureBuilder.dimension; import javax.xml.stream.XMLInputFactory; /** diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java rename to sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java index 47952ed3..023d15ce 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java @@ -14,10 +14,11 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.file; +package internal.file.xml; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.file.SdmxDecoder; import java.io.IOException; import javax.xml.stream.XMLInputFactory; import org.junit.Test; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java index 5e9e27a7..1472c3fb 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/DataFactory.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.parser; -import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.util.Chars; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Frequency; import internal.parser.DataFactories; @@ -32,13 +32,13 @@ public interface DataFactory { @Nonnull - FreqParser getFreqParser(@Nonnull DataStructure dsd); + Freqs.Parser getFreqParser(@Nonnull DataStructure dsd); @Nonnull - SafeParser getPeriodParser(@Nonnull Frequency freq); + Chars.Parser getPeriodParser(@Nonnull Frequency freq); @Nonnull - SafeParser getValueParser(); + Chars.Parser getValueParser(); @Nonnull static DataFactory sdmx20() { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java deleted file mode 100644 index 8a3fc7df..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqParser.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.parser; - -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.Frequency; -import internal.parser.FreqParsers; -import java.util.function.BiFunction; -import java.util.function.Function; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -public interface FreqParser { - - @Nonnull - Frequency parse(@Nonnull Key.Builder key, @Nonnull Function attributes); - - @Nonnull - static FreqParser sdmx20() { - return FreqParsers::parseSdmx20; - } - - @Nonnull - static FreqParser sdmx21(@Nonnull DataStructure dsd) { - return of(FreqUtil.extractorByIndex(dsd), FreqUtil::parseByFreq); - } - - @Nonnull - static FreqParser sdmx21(int frequencyCodeIdIndex) { - return of(FreqUtil.extractorByIndex(frequencyCodeIdIndex), FreqUtil::parseByFreq); - } - - @Nonnull - static FreqParser of( - @Nonnull BiFunction, String> extractor, - @Nonnull Function mapper) { - return (k, a) -> { - String code = extractor.apply(k, a); - if (code == null) { - return Frequency.UNDEFINED; - } - Frequency freq = mapper.apply(code); - if (freq == null) { - return Frequency.UNDEFINED; - } - return freq; - }; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/Freqs.java similarity index 66% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/Freqs.java index 11734fb6..e3c2b8e3 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/FreqUtil.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/Freqs.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.parser; -import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.util.Chars; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Frequency; @@ -35,7 +35,45 @@ * @author Philippe Charles */ @lombok.experimental.UtilityClass -public class FreqUtil { +public class Freqs { + + public interface Parser { + + @Nonnull + Frequency parse(@Nonnull Key.Builder key, @Nonnull Function attributes); + + @Nonnull + static Parser sdmx20() { + return Freqs::parseSdmx20; + } + + @Nonnull + static Parser sdmx21(@Nonnull DataStructure dsd) { + return of(extractorByIndex(dsd), Freqs::parseByFreq); + } + + @Nonnull + static Parser sdmx21(int frequencyCodeIdIndex) { + return of(extractorByIndex(frequencyCodeIdIndex), Freqs::parseByFreq); + } + + @Nonnull + static Parser of( + @Nonnull BiFunction, String> extractor, + @Nonnull Function mapper) { + return (k, a) -> { + String code = extractor.apply(k, a); + if (code == null) { + return Frequency.UNDEFINED; + } + Frequency freq = mapper.apply(code); + if (freq == null) { + return Frequency.UNDEFINED; + } + return freq; + }; + } + } public final String FREQ_CONCEPT = "FREQ"; public final String TIME_FORMAT_CONCEPT = "TIME_FORMAT"; @@ -147,21 +185,21 @@ public Frequency parseByTimeFormat(@Nonnull String code) { } @Nonnull - public static SafeParser onStandardFreq(@Nonnull Frequency freq) { + public static Chars.Parser onStandardFreq(@Nonnull Frequency freq) { return STANDARD_PARSERS.get(freq); } - private final Map> STANDARD_PARSERS = initStandardParsers(); + private final Map> STANDARD_PARSERS = initStandardParsers(); - private Map> initStandardParsers() { - SafeParser yearMonth = SafeParser.onDatePattern("yyyy-MM"); - SafeParser yearMonthDay = SafeParser.onDatePattern("yyyy-MM-dd"); + private Map> initStandardParsers() { + Chars.Parser yearMonth = Chars.Parser.onDatePattern("yyyy-MM"); + Chars.Parser yearMonthDay = Chars.Parser.onDatePattern("yyyy-MM-dd"); - Map> result = new EnumMap<>(Frequency.class); - result.put(ANNUAL, SafeParser.onDatePattern("yyyy").or(SafeParser.onDatePattern("yyyy'-01'")).or(SafeParser.onDatePattern("yyyy'-A1'"))); - result.put(HALF_YEARLY, SafeParser.onYearFreqPos("S", 2).or(yearMonth)); - result.put(QUARTERLY, SafeParser.onYearFreqPos("Q", 4).or(yearMonth)); - result.put(MONTHLY, SafeParser.onYearFreqPos("M", 12).or(yearMonth)); + Map> result = new EnumMap<>(Frequency.class); + result.put(ANNUAL, Chars.Parser.onDatePattern("yyyy").or(Chars.Parser.onDatePattern("yyyy'-01'")).or(Chars.Parser.onDatePattern("yyyy'-A1'"))); + result.put(HALF_YEARLY, Chars.Parser.onYearFreqPos("S", 2).or(yearMonth)); + result.put(QUARTERLY, Chars.Parser.onYearFreqPos("Q", 4).or(yearMonth)); + result.put(MONTHLY, Chars.Parser.onYearFreqPos("M", 12).or(yearMonth)); result.put(WEEKLY, yearMonthDay); result.put(DAILY, yearMonthDay); // FIXME: needs other pattern for time @@ -171,4 +209,9 @@ private Map> initStandardParsers() { result.put(UNDEFINED, yearMonth); return Collections.unmodifiableMap(result); } + + private Frequency parseSdmx20(Key.Builder key, Function attributes) { + String code = attributes.apply(TIME_FORMAT_CONCEPT); + return code != null ? parseByTimeFormat(code) : Frequency.UNDEFINED; + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java index 9724e406..ff9dea83 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/parser/ObsParser.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.parser; -import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.util.Chars; import be.nbb.sdmx.facade.Frequency; import java.time.LocalDateTime; import java.util.function.Function; @@ -31,17 +31,17 @@ public final class ObsParser { @Nonnull public static ObsParser standard() { - return new ObsParser(FreqUtil::onStandardFreq, SafeParser.onStandardDouble()); + return new ObsParser(Freqs::onStandardFreq, Chars.Parser.onStandardDouble()); } - private final Function> toPeriodParser; - private final SafeParser valueParser; - private SafeParser periodParser; + private final Function> toPeriodParser; + private final Chars.Parser valueParser; + private Chars.Parser periodParser; private Frequency freq; private String period; private String value; - public ObsParser(Function> toPeriodParser, SafeParser valueParser) { + public ObsParser(Function> toPeriodParser, Chars.Parser valueParser) { this.toPeriodParser = toPeriodParser; this.valueParser = valueParser; this.periodParser = toPeriodParser.apply(Frequency.UNDEFINED); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/Chars.java similarity index 53% rename from sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/Chars.java index a484ad7d..faa5bcee 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/SafeParsers.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/Chars.java @@ -14,29 +14,80 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.util; +package be.nbb.sdmx.facade.util; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import be.nbb.sdmx.facade.util.SafeParser; +import javax.annotation.Nonnegative; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * * @author Philippe Charles */ @lombok.experimental.UtilityClass -public class SafeParsers { +public class Chars { - public static final class Fallback implements SafeParser { + public interface Parser { - private final SafeParser first; - private final SafeParser second; + @Nullable + T parse(@Nonnull CharSequence input); - public Fallback(SafeParser first, SafeParser second) { + @Nonnull + default Parser or(@Nonnull Parser r) { + return new Fallback(this, r); + } + + @Nonnull + default Parser or(@Nonnull T value) { + return new Fallback(this, o -> value); + } + + @Nonnull + @SuppressWarnings("null") + static Parser onNull() { + return o -> null; + } + + @Nonnull + static Parser onDatePattern(@Nonnull String pattern) { + DateTimeFormatter dateFormat = new DateTimeFormatterBuilder() + .appendPattern(pattern) + .parseStrict() + .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) + .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + .toFormatter(Locale.ROOT); + return new OnDateTimeFormatter(dateFormat); + } + + @Nonnull + static Parser onYearFreqPos(@Nonnull String freqCode, @Nonnegative int freq) { + return new YearFreqPos(freqCode, freq); + } + + @Nonnull + static Parser onStandardDouble() { + return Chars::doubleOrNull; + } + } + + private static final class Fallback implements Parser { + + private final Parser first; + private final Parser second; + + public Fallback(Parser first, Parser second) { this.first = first; this.second = second; } @@ -48,7 +99,7 @@ public T parse(CharSequence input) { } } - public static final class OnDateTimeFormatter implements SafeParser { + private static final class OnDateTimeFormatter implements Parser { private final DateTimeFormatter dateFormat; @@ -66,7 +117,7 @@ public LocalDateTime parse(CharSequence input) { } } - public static final class YearFreqPos implements SafeParser { + private static final class YearFreqPos implements Parser { private final Pattern regex; private final int freq; @@ -87,7 +138,7 @@ private LocalDateTime toDate(int year, int freq, int pos) { } } - public Double doubleOrNull(CharSequence input) { + private Double doubleOrNull(CharSequence input) { try { return Double.valueOf(input.toString()); } catch (NumberFormatException ex) { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java deleted file mode 100644 index db44e052..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SafeParser.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.util; - -import internal.util.SafeParsers; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Locale; -import javax.annotation.Nonnegative; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** - * - * @author Philippe Charles - * @param - */ -public interface SafeParser { - - @Nullable - T parse(@Nonnull CharSequence input); - - @Nonnull - default SafeParser or(@Nonnull SafeParser r) { - return new SafeParsers.Fallback(this, r); - } - - @Nonnull - default SafeParser or(@Nonnull T value) { - return new SafeParsers.Fallback(this, o -> value); - } - - @Nonnull - @SuppressWarnings("null") - static SafeParser onNull() { - return o -> null; - } - - @Nonnull - static SafeParser onDatePattern(@Nonnull String pattern) { - DateTimeFormatter dateFormat = new DateTimeFormatterBuilder() - .appendPattern(pattern) - .parseStrict() - .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) - .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) - .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) - .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) - .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) - .toFormatter(Locale.ROOT); - return new SafeParsers.OnDateTimeFormatter(dateFormat); - } - - @Nonnull - static SafeParser onYearFreqPos(@Nonnull String freqCode, @Nonnegative int freq) { - return new SafeParsers.YearFreqPos(freqCode, freq); - } - - @Nonnull - static SafeParser onStandardDouble() { - return SafeParsers::doubleOrNull; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 61b01658..6ecb0c43 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -29,7 +29,7 @@ import java.util.Map; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.parser.FreqParser; +import be.nbb.sdmx.facade.parser.Freqs; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; @@ -49,14 +49,14 @@ final class XMLStreamCompactDataCursor implements DataCursor { private final Key.Builder keyBuilder; private final AttributesBuilder attributesBuilder; private final ObsParser obsParser; - private final FreqParser freqParser; + private final Freqs.Parser freqParser; private final String timeDimensionId; private final String primaryMeasureId; private boolean closed; private boolean hasSeries; private boolean hasObs; - XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, String timeDimensionId, String primaryMeasureId) { + XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser, String timeDimensionId, String primaryMeasureId) { if (!Stax.isNotNamespaceAware(reader)) { log.fine("Using XMLStreamReader with namespace awareness"); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index cc178979..8ae60459 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -29,7 +29,7 @@ import java.util.Map; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.parser.FreqParser; +import be.nbb.sdmx.facade.parser.Freqs; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; import java.io.Closeable; import java.util.function.BiConsumer; @@ -43,11 +43,11 @@ @lombok.extern.java.Log final class XMLStreamGenericDataCursor implements DataCursor { - static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + static XMLStreamGenericDataCursor sdmx20(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser) { return new XMLStreamGenericDataCursor(reader, onClose, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX20); } - static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser) { + static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser) { return new XMLStreamGenericDataCursor(reader, onClose, keyBuilder, obsParser, freqParser, SeriesHeadParser.SDMX21); } @@ -65,13 +65,13 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClo private final Key.Builder keyBuilder; private final AttributesBuilder attributesBuilder; private final ObsParser obsParser; - private final FreqParser freqParser; + private final Freqs.Parser freqParser; private final SeriesHeadParser headParser; private boolean closed; private boolean hasSeries; private boolean hasObs; - private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, FreqParser freqParser, SeriesHeadParser headParser) { + private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser, SeriesHeadParser headParser) { if (!Stax.isNotNamespaceAware(reader)) { log.fine("Using XMLStreamReader with namespace awareness"); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java index 32907d91..b85a87ff 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/DataFactories.java @@ -19,9 +19,8 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.parser.DataFactory; -import be.nbb.sdmx.facade.parser.FreqParser; -import be.nbb.sdmx.facade.parser.FreqUtil; -import be.nbb.sdmx.facade.util.SafeParser; +import be.nbb.sdmx.facade.parser.Freqs; +import be.nbb.sdmx.facade.util.Chars; import java.time.LocalDateTime; /** @@ -32,34 +31,34 @@ public enum DataFactories implements DataFactory { SDMX20 { @Override - public FreqParser getFreqParser(DataStructure dsd) { - return FreqParser.sdmx20(); + public Freqs.Parser getFreqParser(DataStructure dsd) { + return Freqs.Parser.sdmx20(); } @Override - public SafeParser getPeriodParser(Frequency freq) { - return FreqUtil.onStandardFreq(freq); + public Chars.Parser getPeriodParser(Frequency freq) { + return Freqs.onStandardFreq(freq); } @Override - public SafeParser getValueParser() { - return SafeParser.onStandardDouble(); + public Chars.Parser getValueParser() { + return Chars.Parser.onStandardDouble(); } }, SDMX21 { @Override - public FreqParser getFreqParser(DataStructure dsd) { - return FreqParser.sdmx21(dsd); + public Freqs.Parser getFreqParser(DataStructure dsd) { + return Freqs.Parser.sdmx21(dsd); } @Override - public SafeParser getPeriodParser(Frequency freq) { - return FreqUtil.onStandardFreq(freq); + public Chars.Parser getPeriodParser(Frequency freq) { + return Freqs.onStandardFreq(freq); } @Override - public SafeParser getValueParser() { - return SafeParser.onStandardDouble(); + public Chars.Parser getValueParser() { + return Chars.Parser.onStandardDouble(); } } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java deleted file mode 100644 index cc7f4316..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/internal/parser/FreqParsers.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.parser; - -import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.parser.FreqUtil; -import java.util.function.Function; - -/** - * - * @author Philippe Charles - */ -@lombok.experimental.UtilityClass -public class FreqParsers { - - public Frequency parseSdmx20(Key.Builder key, Function attributes) { - String code = attributes.apply(FreqUtil.TIME_FORMAT_CONCEPT); - return code != null ? FreqUtil.parseByTimeFormat(code) : Frequency.UNDEFINED; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java index 90c121bf..618730e6 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.parser; -import be.nbb.sdmx.facade.parser.FreqUtil; +import be.nbb.sdmx.facade.parser.Freqs; import be.nbb.sdmx.facade.Frequency; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -32,24 +32,24 @@ public class FreqUtilTest { @Test @SuppressWarnings("null") public void testParseByFreq() { - assertThatThrownBy(() -> FreqUtil.parseByFreq(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> Freqs.parseByFreq(null)).isInstanceOf(NullPointerException.class); - assertThat(FreqUtil.parseByFreq("A")).isEqualTo(Frequency.ANNUAL); - assertThat(FreqUtil.parseByFreq("S")).isEqualTo(Frequency.HALF_YEARLY); - assertThat(FreqUtil.parseByFreq("Q")).isEqualTo(Frequency.QUARTERLY); - assertThat(FreqUtil.parseByFreq("M")).isEqualTo(Frequency.MONTHLY); - assertThat(FreqUtil.parseByFreq("W")).isEqualTo(Frequency.WEEKLY); - assertThat(FreqUtil.parseByFreq("D")).isEqualTo(Frequency.DAILY); - assertThat(FreqUtil.parseByFreq("H")).isEqualTo(Frequency.HOURLY); - assertThat(FreqUtil.parseByFreq("B")).isEqualTo(Frequency.DAILY_BUSINESS); - assertThat(FreqUtil.parseByFreq("N")).isEqualTo(Frequency.MINUTELY); + assertThat(Freqs.parseByFreq("A")).isEqualTo(Frequency.ANNUAL); + assertThat(Freqs.parseByFreq("S")).isEqualTo(Frequency.HALF_YEARLY); + assertThat(Freqs.parseByFreq("Q")).isEqualTo(Frequency.QUARTERLY); + assertThat(Freqs.parseByFreq("M")).isEqualTo(Frequency.MONTHLY); + assertThat(Freqs.parseByFreq("W")).isEqualTo(Frequency.WEEKLY); + assertThat(Freqs.parseByFreq("D")).isEqualTo(Frequency.DAILY); + assertThat(Freqs.parseByFreq("H")).isEqualTo(Frequency.HOURLY); + assertThat(Freqs.parseByFreq("B")).isEqualTo(Frequency.DAILY_BUSINESS); + assertThat(Freqs.parseByFreq("N")).isEqualTo(Frequency.MINUTELY); - assertThat(FreqUtil.parseByFreq("A5")).isEqualTo(Frequency.ANNUAL); - assertThat(FreqUtil.parseByFreq("M2")).isEqualTo(Frequency.MONTHLY); - assertThat(FreqUtil.parseByFreq("W6")).isEqualTo(Frequency.WEEKLY); + assertThat(Freqs.parseByFreq("A5")).isEqualTo(Frequency.ANNUAL); + assertThat(Freqs.parseByFreq("M2")).isEqualTo(Frequency.MONTHLY); + assertThat(Freqs.parseByFreq("W6")).isEqualTo(Frequency.WEEKLY); - assertThat(FreqUtil.parseByFreq("")).isEqualTo(Frequency.UNDEFINED); - assertThat(FreqUtil.parseByFreq("A0")).isEqualTo(Frequency.UNDEFINED); - assertThat(FreqUtil.parseByFreq("A1")).isEqualTo(Frequency.UNDEFINED); + assertThat(Freqs.parseByFreq("")).isEqualTo(Frequency.UNDEFINED); + assertThat(Freqs.parseByFreq("A0")).isEqualTo(Frequency.UNDEFINED); + assertThat(Freqs.parseByFreq("A1")).isEqualTo(Frequency.UNDEFINED); } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index b5e08d87..dec2ec90 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -19,14 +19,14 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.parser.Freqs; import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; import be.nbb.sdmx.facade.parser.ObsParser; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import be.nbb.sdmx.facade.parser.FreqParser; import java.io.InputStream; import javax.xml.stream.XMLInputFactory; @@ -43,11 +43,11 @@ public void testCompactData20() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE"); + return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx20(), "TIME_PERIOD", "OBS_VALUE"); }); try (InputStream stream = xml.openStream(); - DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { + DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx20(), "TIME_PERIOD", "OBS_VALUE")) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -87,11 +87,11 @@ public void testCompactData21() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE"); + return new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE"); }); try (InputStream stream = xml.openStream(); - DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { + DataCursor o = new XMLStreamCompactDataCursor(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0), "TIME_PERIOD", "OBS_VALUE")) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 0bcad1bb..ed89dc60 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -19,14 +19,14 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.parser.Freqs; import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; import be.nbb.sdmx.facade.parser.ObsParser; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import be.nbb.sdmx.facade.parser.FreqParser; import java.io.InputStream; import javax.xml.stream.XMLInputFactory; @@ -43,11 +43,11 @@ public void testGenericData20() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20()); + return XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx20()); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx20())) { + DataCursor o = XMLStreamGenericDataCursor.sdmx20(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx20())) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { @@ -91,11 +91,11 @@ public void testGenericData21() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0)); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0))) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(Key.of("A", "BEL", "1", "0", "0", "0", "OVGD")); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); @@ -131,11 +131,11 @@ public void testGenericData21Bis() throws Exception { DataCursorAssert.assertCompliance(() -> { InputStream stream = xml.openStream(); - return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0)); + return XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0)); }); try (InputStream stream = xml.openStream(); - DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), FreqParser.sdmx21(0))) { + DataCursor o = XMLStreamGenericDataCursor.sdmx21(xif.createXMLStreamReader(stream), stream, builder, ObsParser.standard(), Freqs.Parser.sdmx21(0))) { int indexSeries = -1; while (o.nextSeries()) { switch (++indexSeries) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index a8a7175d..f1225ee3 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -27,8 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import be.nbb.sdmx.facade.parser.FreqUtil; -import static be.nbb.sdmx.facade.parser.FreqUtil.TIME_FORMAT_CONCEPT; +import be.nbb.sdmx.facade.parser.Freqs; +import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; /** * @@ -137,11 +137,11 @@ private void checkObsState() throws IOException, IllegalStateException { // private static Frequency getFrequency(PortableTimeSeries input) { if (input.getFrequency() != null) { - return FreqUtil.parseByFreq(input.getFrequency()); + return Freqs.parseByFreq(input.getFrequency()); } String value = input.getAttribute(TIME_FORMAT_CONCEPT); if (value != null) { - return FreqUtil.parseByTimeFormat(value); + return Freqs.parseByTimeFormat(value); } return Frequency.UNDEFINED; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java index c75c1bc6..d299c6b0 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java @@ -22,9 +22,8 @@ import static be.nbb.sdmx.facade.Frequency.HALF_YEARLY; import static be.nbb.sdmx.facade.Frequency.MONTHLY; import static be.nbb.sdmx.facade.Frequency.QUARTERLY; -import be.nbb.sdmx.facade.parser.FreqParser; -import be.nbb.sdmx.facade.util.SafeParser; -import be.nbb.sdmx.facade.parser.FreqUtil; +import be.nbb.sdmx.facade.parser.Freqs; +import be.nbb.sdmx.facade.util.Chars; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import java.time.LocalDateTime; import org.openide.util.lookup.ServiceProvider; @@ -48,18 +47,18 @@ public String getDescription() { } @Override - public FreqParser getFreqParser(DataStructure dsd) { - return FreqParser.of(FreqUtil.extractorByIndex(dsd), InseeDialect::parseInseeFreq); + public Freqs.Parser getFreqParser(DataStructure dsd) { + return Freqs.Parser.of(Freqs.extractorByIndex(dsd), InseeDialect::parseInseeFreq); } @Override - public SafeParser getPeriodParser(Frequency freq) { + public Chars.Parser getPeriodParser(Frequency freq) { return onInseeTimePeriod(freq); } @Override - public SafeParser getValueParser() { - return SafeParser.onStandardDouble(); + public Chars.Parser getValueParser() { + return Chars.Parser.onStandardDouble(); } private static Frequency parseInseeFreq(String code) { @@ -80,7 +79,7 @@ private static Frequency parseInseeFreq(String code) { return Frequency.UNDEFINED; } - private static SafeParser onInseeTimePeriod(Frequency freq) { + private static Chars.Parser onInseeTimePeriod(Frequency freq) { switch (freq) { case ANNUAL: return ANNUAL_PARSER; @@ -95,9 +94,9 @@ private static SafeParser onInseeTimePeriod(Frequency freq) { } } - private static final SafeParser ANNUAL_PARSER = SafeParser.onDatePattern("yyyy"); - private static final SafeParser HALF_YEARLY_PARSER = SafeParser.onYearFreqPos("S", 2); - private static final SafeParser QUARTERLY_PARSER = SafeParser.onYearFreqPos("Q", 4); - private static final SafeParser MONTHLY_PARSER = SafeParser.onDatePattern("yyyy-MM").or(SafeParser.onYearFreqPos("B", 12)); - private static final SafeParser DEFAULT_PARSER = SafeParser.onNull(); + private static final Chars.Parser ANNUAL_PARSER = Chars.Parser.onDatePattern("yyyy"); + private static final Chars.Parser HALF_YEARLY_PARSER = Chars.Parser.onYearFreqPos("S", 2); + private static final Chars.Parser QUARTERLY_PARSER = Chars.Parser.onYearFreqPos("Q", 4); + private static final Chars.Parser MONTHLY_PARSER = Chars.Parser.onDatePattern("yyyy-MM").or(Chars.Parser.onYearFreqPos("B", 12)); + private static final Chars.Parser DEFAULT_PARSER = Chars.Parser.onNull(); } From fcce2d64db2a6e78a88c8f6aeb7d21eb8ac91dd3 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 21 Nov 2017 17:30:03 +0100 Subject: [PATCH 50/68] Added code coverage. --- sdmx-facade/pom.xml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index 25c5a2dc..a8ed233f 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -25,6 +25,7 @@ 1.16.18 4.12 3.8.0 + 0.7.9 @@ -78,4 +79,28 @@ + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + + prepare-agent + + + + report + prepare-package + + report + + + + + + \ No newline at end of file From 838b2a1ea950d7e575f22556041161d29b19c0ed Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 22 Nov 2017 10:48:21 +0100 Subject: [PATCH 51/68] Improved code coverage. Code cleanup. --- ...{FlowRefTest.java => DataflowRefTest.java} | 15 ++- .../be/nbb/sdmx/facade/DimentionTest.java | 38 ++---- .../sdmx/facade/LanguagePriorityListTest.java | 42 +++---- .../nbb/sdmx/facade/file/SdmxFileManager.java | 11 +- .../sdmx/facade/file/SdmxFileManagerTest.java | 12 +- .../file/SdmxFileConnectionImplTest.java | 4 +- .../java/internal/file/SdmxFileUtilTest.java | 11 +- .../be/nbb/sdmx/facade/util/HasCache.java | 17 +++ .../main/java/be/nbb/sdmx/facade/util/IO.java | 30 ++++- .../java/internal/util/HasCacheSupport.java | 49 ++++++++ .../{FreqUtilTest.java => FreqsTest.java} | 9 +- .../sdmx/facade/util/SeriesSupportTest.java | 23 ++-- .../nbb/sdmx/facade/xml/stream/StaxTest.java | 20 +-- .../facade/xml/stream/TextBuilderTest.java | 9 +- .../xml/stream/XMLStreamStructure20Test.java | 13 +- .../xml/stream/XMLStreamStructure21Test.java | 12 +- sdmx-facade/sdmx-facade-web/pom.xml | 18 +++ .../nbb/sdmx/facade/web/SdmxWebManager.java | 17 ++- .../connectors/ConnectorsDriverSupport.java | 114 ++++++++++++------ .../connectors/GenericSDMXClientSupplier.java | 48 -------- .../connectors/PortableTimeSeriesCursor.java | 4 +- .../main/java/internal/connectors/Util.java | 2 +- .../connectors/drivers/AbsDriver.java | 16 +-- .../connectors/drivers/EurostatDriver.java | 16 +-- .../connectors/drivers/IloDriver.java | 16 +-- .../connectors/drivers/ImfDriver.java | 16 +-- .../connectors/drivers/InseeDriver.java | 24 ++-- .../connectors/drivers/NbbDriver.java | 16 +-- .../connectors/drivers/OecdDriver.java | 16 +-- .../connectors/drivers/Sdmx20Driver.java | 21 +--- .../connectors/drivers/Sdmx21Driver.java | 21 ++-- .../connectors/drivers/UisDriver.java | 16 +-- .../internal/util/drivers/InseeDialect.java | 1 + .../sdmx/facade/web/SdmxWebManagerTest.java | 13 +- .../connectors/CachedResourceTest.java | 8 +- .../ConnectorsDriverSupportTest.java | 30 +++++ .../PortableTimeSeriesCursorTest.java | 1 + .../java/internal/connectors/UtilTest.java | 35 ++++++ .../connectors/drivers/AbsDriverTest.java | 32 +++++ .../drivers/EurostatDriverTest.java | 32 +++++ .../connectors/drivers/IloDriverTest.java | 32 +++++ .../connectors/drivers/ImfDriverTest.java | 32 +++++ .../connectors/drivers/InseeDriverTest.java | 32 +++++ .../connectors/drivers/NbbDriverTest.java | 32 +++++ .../connectors/drivers/OecdDriverTest.java | 32 +++++ .../connectors/drivers/Sdmx20DriverTest.java | 42 +++++++ .../connectors/drivers/Sdmx21DriverTest.java | 32 +++++ .../connectors/drivers/UisDriverTest.java | 32 +++++ .../util/drivers/InseeDialectTest.java | 76 ++++++++++++ .../src/test/java/test/CacheAssertions.java | 39 ++++++ .../ConnectorsResource.java | 8 +- .../src/test/java/test/DialectAssertions.java | 39 ++++++ .../src/test/java/test/DriverAssertions.java | 58 +++++++++ .../connectors => test}/FacadeResource.java | 3 +- .../connectors => test}/ParsersTest.java | 2 +- 55 files changed, 990 insertions(+), 349 deletions(-) rename sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/{FlowRefTest.java => DataflowRefTest.java} (83%) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/internal/util/HasCacheSupport.java rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/{FreqUtilTest.java => FreqsTest.java} (85%) delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/CacheAssertions.java rename sdmx-facade/sdmx-facade-web/src/test/java/{internal/connectors => test}/ConnectorsResource.java (93%) create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/DialectAssertions.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java rename sdmx-facade/sdmx-facade-web/src/test/java/{internal/connectors => test}/FacadeResource.java (99%) rename sdmx-facade/sdmx-facade-web/src/test/java/{internal/connectors => test}/ParsersTest.java (97%) diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java similarity index 83% rename from sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java rename to sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java index d1c63309..883932ec 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/FlowRefTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java @@ -18,15 +18,14 @@ import static be.nbb.sdmx.facade.DataflowRef.ALL_AGENCIES; import static be.nbb.sdmx.facade.DataflowRef.LATEST_VERSION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** * * @author Philippe Charles */ -public class FlowRefTest { +public class DataflowRefTest { @Test @SuppressWarnings("null") @@ -38,8 +37,8 @@ public void testParse() { assertThat(DataflowRef.parse("world,hello,")).isEqualTo(DataflowRef.of("world", "hello", LATEST_VERSION)); assertThat(DataflowRef.parse(",hello,")).isEqualTo(DataflowRef.of(ALL_AGENCIES, "hello", LATEST_VERSION)); assertThat(DataflowRef.parse(",,")).isEqualTo(DataflowRef.of(ALL_AGENCIES, "", LATEST_VERSION)); - assertThatThrownBy(() -> DataflowRef.parse(",,,,")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> DataflowRef.parse(null)).isInstanceOf(NullPointerException.class); + assertThatIllegalArgumentException().isThrownBy(() -> DataflowRef.parse(",,,,")); + assertThatNullPointerException().isThrownBy(() -> DataflowRef.parse(null)); } @Test @@ -61,8 +60,8 @@ public void testValueOf() { .extracting(DataflowRef::getAgency, DataflowRef::getId, DataflowRef::getVersion, DataflowRef::toString) .containsExactly("world", "hello", "123", "world,hello,123"); - assertThatThrownBy(() -> DataflowRef.of(null, "world,hello", null)).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> DataflowRef.of(null, null, null)).isInstanceOf(NullPointerException.class); + assertThatIllegalArgumentException().isThrownBy(() -> DataflowRef.of(null, "world,hello", null)); + assertThatNullPointerException().isThrownBy(() -> DataflowRef.of(null, null, null)); } @Test @@ -81,6 +80,6 @@ public void testContains() { assertThat(DataflowRef.of("world", "hello", "123").contains(DataflowRef.of(ALL_AGENCIES, "hello", "123"))).isFalse(); assertThat(DataflowRef.of("world", "hello", LATEST_VERSION).contains(DataflowRef.of("world", "hello", "123"))).isTrue(); assertThat(DataflowRef.of("world", "hello", "123").contains(DataflowRef.of("world", "hello", LATEST_VERSION))).isFalse(); - assertThatThrownBy(() -> DataflowRef.of("world", "hello", "123").contains(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> DataflowRef.of("world", "hello", "123").contains(null)); } } diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java index 0d15d866..8b004017 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java @@ -16,9 +16,9 @@ */ package be.nbb.sdmx.facade; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static be.nbb.sdmx.facade.Dimension.builder; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; /** * @@ -31,41 +31,27 @@ public class DimentionTest { @Test public void testBuilder() { - assertThatThrownBy(Dimension.builder()::build) - .as("Codelist#getId() must be non-null") - .isInstanceOf(NullPointerException.class) - .hasMessage("id"); + assertThatNullPointerException().isThrownBy(builder()::build).withMessage("id"); + assertThatNullPointerException().isThrownBy(builder().id(null)::build).withMessage("id"); + assertThatNullPointerException().isThrownBy(builder().id(someId).label(null)::build).withMessage("label"); + assertThatNullPointerException().isThrownBy(() -> builder().codes(null)); - assertThatThrownBy(Dimension.builder().id(null)::build) - .as("Codelist#getId() must be non-null") - .isInstanceOf(NullPointerException.class) - .hasMessage("id"); + assertThatExceptionOfType(UnsupportedOperationException.class) + .isThrownBy(() -> builder().id(someId).label(someLabel).build().getCodes().put("hello", "world")); - assertThatThrownBy(Dimension.builder().id(someId).label(null)::build) - .as("Codelist#getLabel() must be non-null") - .isInstanceOf(NullPointerException.class); - - assertThatThrownBy(() -> Dimension.builder().codes(null)) - .as("Codelist#getCodes() must be non-null") - .isInstanceOf(NullPointerException.class); - - assertThatThrownBy(() -> Dimension.builder().id(someId).label(someLabel).build().getCodes().put("hello", "world")) - .as("Codelist#getCodes() must return immutable map") - .isInstanceOf(UnsupportedOperationException.class); - - assertThat(Dimension.builder().id(someId).label(someLabel).build()) + assertThat(builder().id(someId).label(someLabel).build()) .hasFieldOrPropertyWithValue("id", someId) .hasFieldOrPropertyWithValue("label", someLabel) .hasNoNullFieldsOrProperties(); - assertThat(Dimension.builder().id(someId).label(someLabel).code("hello", "world").build().getCodes()) + assertThat(builder().id(someId).label(someLabel).code("hello", "world").build().getCodes()) .containsEntry("hello", "world") .hasSize(1); } @Test public void testEquals() { - assertThat(Dimension.builder().id("id").label("label").position(1).code("k1", "v1").code("k2", "v2").build()) - .isEqualTo(Dimension.builder().id("id").label("label").position(1).code("k2", "v2").code("k1", "v1").build()); + assertThat(builder().id("id").label("label").position(1).code("k1", "v1").code("k2", "v2").build()) + .isEqualTo(builder().id("id").label("label").position(1).code("k2", "v2").code("k1", "v1").build()); } } diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/LanguagePriorityListTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/LanguagePriorityListTest.java index 75f9367d..31658652 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/LanguagePriorityListTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/LanguagePriorityListTest.java @@ -16,11 +16,11 @@ */ package be.nbb.sdmx.facade; +import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; +import static be.nbb.sdmx.facade.LanguagePriorityList.parse; import java.util.Arrays; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; /** * @@ -31,33 +31,33 @@ public class LanguagePriorityListTest { @Test @SuppressWarnings("null") public void testParse() { - assertThat(LanguagePriorityList.parse("*")).hasToString("*"); - assertThat(LanguagePriorityList.parse("fr")).hasToString("fr"); - assertThat(LanguagePriorityList.parse("fr-BE")).hasToString("fr-be"); - assertThat(LanguagePriorityList.parse("fr-BE,fr;q=0.5")).hasToString("fr-be,fr;q=0.5"); - assertThat(LanguagePriorityList.parse("fr-CH, fr;q=0.9, en;q=0.8, de;q=0.7, *;q=0.5")).hasToString("fr-ch,fr;q=0.9,en;q=0.8,de;q=0.7,*;q=0.5"); - assertThatThrownBy(() -> LanguagePriorityList.parse("fr-BE;")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> LanguagePriorityList.parse(null)).isInstanceOf(NullPointerException.class); + assertThat(parse("*")).hasToString("*"); + assertThat(parse("fr")).hasToString("fr"); + assertThat(parse("fr-BE")).hasToString("fr-be"); + assertThat(parse("fr-BE,fr;q=0.5")).hasToString("fr-be,fr;q=0.5"); + assertThat(parse("fr-CH, fr;q=0.9, en;q=0.8, de;q=0.7, *;q=0.5")).hasToString("fr-ch,fr;q=0.9,en;q=0.8,de;q=0.7,*;q=0.5"); + assertThatIllegalArgumentException().isThrownBy(() -> parse("fr-BE;")); + assertThatNullPointerException().isThrownBy(() -> parse(null)); } @Test public void testEquals() { - assertThat(LanguagePriorityList.parse("*")) - .isEqualTo(LanguagePriorityList.parse("*")) - .isEqualTo(LanguagePriorityList.ANY); + assertThat(parse("*")) + .isEqualTo(parse("*")) + .isEqualTo(ANY); - assertThat(LanguagePriorityList.parse("fr-BE")) - .isEqualTo(LanguagePriorityList.parse("fr-BE;q=1")) - .isEqualTo(LanguagePriorityList.parse("fr-BE")); + assertThat(parse("fr-BE")) + .isEqualTo(parse("fr-BE;q=1")) + .isEqualTo(parse("fr-BE")); } @Test @SuppressWarnings("null") public void testLookupTag() { - assertThat(LanguagePriorityList.parse("fr").lookupTag(Arrays.asList("fr", "nl"))).isEqualTo("fr"); - assertThat(LanguagePriorityList.parse("fr-BE").lookupTag(Arrays.asList("fr", "nl"))).isEqualTo("fr"); - assertThat(LanguagePriorityList.parse("fr,nl;q=0.7,en;q=0.3").lookupTag(Arrays.asList("de", "nl", "en"))).isEqualTo("nl"); - assertThat(LanguagePriorityList.parse("fr").lookupTag(Arrays.asList("nl"))).isNull(); - assertThatThrownBy(() -> LanguagePriorityList.parse("fr").lookupTag(null)).isInstanceOf(NullPointerException.class); + assertThat(parse("fr").lookupTag(Arrays.asList("fr", "nl"))).isEqualTo("fr"); + assertThat(parse("fr-BE").lookupTag(Arrays.asList("fr", "nl"))).isEqualTo("fr"); + assertThat(parse("fr,nl;q=0.7,en;q=0.3").lookupTag(Arrays.asList("de", "nl", "en"))).isEqualTo("nl"); + assertThat(parse("fr").lookupTag(Arrays.asList("nl"))).isNull(); + assertThatNullPointerException().isThrownBy(() -> parse("fr").lookupTag(null)); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index b7ab7858..8a4a2d9b 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -36,7 +36,6 @@ import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.xml.stream.XMLInputFactory; import lombok.AccessLevel; @@ -56,7 +55,7 @@ public static SdmxFileManager ofServiceLoader() { return new SdmxFileManager( factoryWithoutNamespace, new StaxSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), - new AtomicReference<>(new ConcurrentHashMap()), + HasCache.of(ConcurrentHashMap::new), dialects ); } @@ -65,7 +64,7 @@ public static SdmxFileManager ofServiceLoader() { private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; - private final AtomicReference cache; + private final HasCache cacheSupport; private final List dialects; @Override @@ -90,12 +89,12 @@ public SdmxFileConnection getConnection(@Nonnull SdmxFileSet files, @Nonnull Lan @Override public ConcurrentMap getCache() { - return cache.get(); + return cacheSupport.getCache(); } @Override public void setCache(ConcurrentMap cache) { - this.cache.set(cache != null ? cache : new ConcurrentHashMap()); + this.cacheSupport.setCache(cache); } private SdmxFileSet getFiles(String name) throws IOException { @@ -107,7 +106,7 @@ private SdmxFileSet getFiles(String name) throws IOException { } private SdmxFileConnectionImpl.Resource getResource(SdmxFileSet files, LanguagePriorityList languages) { - return new CachedResource(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), cache.get()); + return new CachedResource(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), getCache()); } private Dataflow getDataflow(SdmxFileSet files) { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java index 14316069..c5d5021f 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/be/nbb/sdmx/facade/file/SdmxFileManagerTest.java @@ -22,7 +22,7 @@ import internal.file.SdmxFileUtil; import java.io.File; import java.io.IOException; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -50,11 +50,11 @@ public void testCompliance() throws IOException { @SuppressWarnings("null") public void test() { SdmxFileManager m = SdmxFileManager.ofServiceLoader(); - assertThatThrownBy(() -> m.getConnection((String) null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> m.getConnection((String) null, LanguagePriorityList.ANY)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> m.getConnection(SdmxFileUtil.toXml(files), null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> m.getConnection((SdmxFileSet) null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> m.getConnection(files, null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> m.getConnection((String) null)); + assertThatNullPointerException().isThrownBy(() -> m.getConnection((String) null, LanguagePriorityList.ANY)); + assertThatNullPointerException().isThrownBy(() -> m.getConnection(SdmxFileUtil.toXml(files), null)); + assertThatNullPointerException().isThrownBy(() -> m.getConnection((SdmxFileSet) null)); + assertThatNullPointerException().isThrownBy(() -> m.getConnection(files, null)); } private final SdmxFileSet files = SdmxFileSet.builder().data(new File("hello")).build(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index d7b217eb..96f42502 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -59,8 +59,8 @@ public void testFile() throws IOException { assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); assertThat(conn.getFlow()).isEqualTo(conn.getFlow(files.asDataflowRef())); assertThat(conn.getStructure()).isEqualTo(conn.getStructure(files.asDataflowRef())); - assertThatThrownBy(() -> conn.getCursor(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> conn.getStream(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> conn.getCursor(null)); + assertThatNullPointerException().isThrownBy(() -> conn.getStream(null)); assertThat(conn.getStream(DataQuery.of(Key.ALL, false))).containsExactly(conn.getStream(DataQuery.of(Key.ALL, false)).toArray(Series[]::new)); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java index 7dce2cc1..2f535456 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileUtilTest.java @@ -18,8 +18,7 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import java.io.File; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -43,10 +42,10 @@ public void testToXml() { @Test @SuppressWarnings("null") public void testFromXml() { - assertThatThrownBy(() -> SdmxFileUtil.fromXml(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SdmxFileUtil.fromXml("")).isInstanceOf(IllegalArgumentException.class); + assertThatNullPointerException().isThrownBy(() -> SdmxFileUtil.fromXml(null)); + assertThatIllegalArgumentException().isThrownBy(() -> SdmxFileUtil.fromXml("")); + assertThatIllegalArgumentException().isThrownBy(() -> SdmxFileUtil.fromXml("")); + assertThatIllegalArgumentException().isThrownBy(() -> SdmxFileUtil.fromXml("")); assertThat(SdmxFileUtil.fromXml("")) .hasFieldOrPropertyWithValue("data", data) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/HasCache.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/HasCache.java index c1ff04cb..d4527cef 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/HasCache.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/HasCache.java @@ -16,7 +16,11 @@ */ package be.nbb.sdmx.facade.util; +import internal.util.HasCacheSupport; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Supplier; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -30,4 +34,17 @@ public interface HasCache { ConcurrentMap getCache(); void setCache(@Nullable ConcurrentMap cache); + + @Nonnull + static HasCache of(@Nonnull Supplier defaultCache) { + return of(defaultCache, (oldObj, newObj) -> { + }); + } + + @Nonnull + static HasCache of(@Nonnull Supplier defaultCache, @Nonnull BiConsumer onCacheChange) { + ConcurrentMap cache = defaultCache.get(); + onCacheChange.accept(null, cache); + return new HasCacheSupport(defaultCache, new AtomicReference<>(cache), onCacheChange); + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java index 31820e8c..3d55751a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java @@ -46,6 +46,12 @@ public interface Function { R applyWithIO(T t) throws IOException; } + @FunctionalInterface + public interface Predicate { + + boolean testWithIO(T t) throws IOException; + } + @Nonnull public Stream stream(IO.Supplier supplier, IO.Function> stream) throws IOException { T resource = supplier.getWithIO(); @@ -98,10 +104,30 @@ public T next() { iter, Spliterator.ORDERED | Spliterator.NONNULL), false); } - private Runnable asUncheckedRunnable(Closeable c) { + private Runnable asUncheckedRunnable(Closeable o) { return () -> { try { - c.close(); + o.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + public java.util.function.Function asUnchecked(Function o) { + return (T t) -> { + try { + return o.applyWithIO(t); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + public java.util.function.Predicate asUnchecked(Predicate o) { + return (T t) -> { + try { + return o.testWithIO(t); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/HasCacheSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/HasCacheSupport.java new file mode 100644 index 00000000..08eedbca --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/internal/util/HasCacheSupport.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util; + +import be.nbb.sdmx.facade.util.HasCache; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor +public final class HasCacheSupport implements HasCache { + + private final Supplier defaultCache; + private final AtomicReference cache; + private final BiConsumer onCacheChange; + + @Override + public ConcurrentMap getCache() { + return cache.get(); + } + + @Override + public void setCache(ConcurrentMap cache) { + ConcurrentMap oldObj = this.cache.get(); + ConcurrentMap newObj = cache != null ? cache : defaultCache.get(); + if (this.cache.compareAndSet(oldObj, newObj)) { + onCacheChange.accept(oldObj, newObj); + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqsTest.java similarity index 85% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqsTest.java index 618730e6..efa025cf 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqUtilTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/parser/FreqsTest.java @@ -16,23 +16,20 @@ */ package be.nbb.sdmx.facade.parser; -import be.nbb.sdmx.facade.parser.Freqs; import be.nbb.sdmx.facade.Frequency; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; /** * * @author Philippe Charles */ -public class FreqUtilTest { +public class FreqsTest { @Test @SuppressWarnings("null") public void testParseByFreq() { - assertThatThrownBy(() -> Freqs.parseByFreq(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> Freqs.parseByFreq(null)); assertThat(Freqs.parseByFreq("A")).isEqualTo(Frequency.ANNUAL); assertThat(Freqs.parseByFreq("S")).isEqualTo(Frequency.HALF_YEARLY); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 1d038369..171542b8 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; +import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.samples.SdmxSource; @@ -33,8 +33,7 @@ import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -46,8 +45,8 @@ public class SeriesSupportTest { @Test @SuppressWarnings("null") public void testAsCursor() { - assertThatThrownBy(() -> SeriesSupport.asCursor(null, Key.ALL)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SeriesSupport.asCursor(Collections.emptyList(), null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> SeriesSupport.asCursor(null, Key.ALL)); + assertThatNullPointerException().isThrownBy(() -> SeriesSupport.asCursor(Collections.emptyList(), null)); DataCursorAssert.assertCompliance(() -> SeriesSupport.asCursor(Collections.emptyList(), Key.ALL)); } @@ -55,19 +54,19 @@ public void testAsCursor() { @Test @SuppressWarnings("null") public void testAsStream() { - assertThatThrownBy(() -> SeriesSupport.asStream(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> SeriesSupport.asStream(null)); - assertThatThrownBy(() -> { + assertThatIllegalStateException().isThrownBy(() -> { DataCursor cursor = NoOpCursor.noOp(); SeriesSupport.asStream(() -> cursor).count(); cursor.getSeriesKey(); - }).isInstanceOf(IllegalStateException.class); + }); } @Test @SuppressWarnings("null") public void testCopyOf() throws IOException, XMLStreamException { - assertThatThrownBy(() -> SeriesSupport.copyOf(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> SeriesSupport.copyOf(null)); try (DataCursor c = NoOpCursor.noOp()) { assertThat(SeriesSupport.copyOf(c)).isEmpty(); @@ -77,7 +76,7 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); + List dsds = SdmxXmlStreams.struct21(ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } @@ -86,7 +85,7 @@ public void testCopyOf() throws IOException, XMLStreamException { @Test @SuppressWarnings("null") public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { - assertThatThrownBy(() -> SeriesSupport.copyOfKeysAndMeta(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> SeriesSupport.copyOfKeysAndMeta(null)); try (DataCursor c = NoOpCursor.noOp()) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)).isEmpty(); @@ -100,7 +99,7 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(LanguagePriorityList.ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); + List dsds = SdmxXmlStreams.struct21(ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java index 01b19f9f..6f8c26df 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java @@ -61,15 +61,9 @@ public void testValidParseStream() throws IOException { return ""; }; - assertThatThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)) - .isInstanceOf(NullPointerException.class) - .hasNoSuppressedExceptions() - .hasNoCause(); + assertThatNullPointerException().isThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)); - assertThatThrownBy(() -> p.parseStream(xif, null, UTF_8)) - .isInstanceOf(NullPointerException.class) - .hasNoSuppressedExceptions() - .hasNoCause(); + assertThatNullPointerException().isThrownBy(() -> p.parseStream(xif, null, UTF_8)); assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) .isInstanceOf(OpenError.class) @@ -97,15 +91,9 @@ public void testInvalidParseStream() throws IOException { } }; - assertThatThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)) - .isInstanceOf(NullPointerException.class) - .hasNoSuppressedExceptions() - .hasNoCause(); + assertThatNullPointerException().isThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)); - assertThatThrownBy(() -> p.parseStream(xif, null, UTF_8)) - .isInstanceOf(NullPointerException.class) - .hasNoSuppressedExceptions() - .hasNoCause(); + assertThatNullPointerException().isThrownBy(() -> p.parseStream(xif, null, UTF_8)); assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) .isInstanceOf(OpenError.class) diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java index b699dc8f..15e8eb3a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java @@ -18,8 +18,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -38,9 +37,9 @@ public void test() { assertThat(new TextBuilder(ANY).put("en", "hello").put("fr", "bonjour").build()).isEqualTo("hello"); assertThat(new TextBuilder(ANY).put("fr", "bonjour").put("en", "hello").build()).isEqualTo("bonjour"); - assertThatThrownBy(() -> new TextBuilder(null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> new TextBuilder(ANY).put(null, "hello")).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> new TextBuilder(ANY).put("en", "hello").build(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> new TextBuilder(null)); + assertThatNullPointerException().isThrownBy(() -> new TextBuilder(ANY).put(null, "hello")); + assertThatNullPointerException().isThrownBy(() -> new TextBuilder(ANY).put("en", "hello").build(null)); assertThat(new TextBuilder(LanguagePriorityList.parse("fr")).put("en", "hello").put("fr", "bonjour").build()).isEqualTo("bonjour"); assertThat(new TextBuilder(LanguagePriorityList.parse("en")).put("fr", "bonjour").put("en", "hello").build()).isEqualTo("hello"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index e54a8b0a..4d373208 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -24,8 +24,7 @@ import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -65,11 +64,11 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> p1.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)) - .isInstanceOf(IOException.class) - .hasCauseInstanceOf(XMLStreamException.class) - .hasMessageContaining("Invalid namespace"); + assertThatIOException() + .isThrownBy(() -> p1.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)) + .withCauseInstanceOf(XMLStreamException.class) + .withMessageContaining("Invalid namespace"); } - + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index f00b31af..fdc95205 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -20,12 +20,10 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; -import java.io.IOException; import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -51,10 +49,10 @@ public void test() throws Exception { }); }); - assertThatThrownBy(() -> parser.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)) - .isInstanceOf(IOException.class) - .hasCauseInstanceOf(XMLStreamException.class) - .hasMessageContaining("Invalid namespace"); + assertThatIOException() + .isThrownBy(() -> parser.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)) + .withCauseInstanceOf(XMLStreamException.class) + .withMessageContaining("Invalid namespace"); } private final XMLInputFactory xif = Stax.getInputFactory(); diff --git a/sdmx-facade/sdmx-facade-web/pom.xml b/sdmx-facade/sdmx-facade-web/pom.xml index 592d8d55..4566fb6b 100644 --- a/sdmx-facade/sdmx-facade-web/pom.xml +++ b/sdmx-facade/sdmx-facade-web/pom.xml @@ -89,6 +89,24 @@ sdmx-facade-samples test + + org.seleniumhq.selenium + selenium-java + test + 2.44.0 + + + com.opera + operadriver + test + 1.5 + + + org.seleniumhq.selenium + selenium-remote-driver + + + sdmx-facade-web \ No newline at end of file diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 950a0ee2..8794b9b3 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -32,7 +32,6 @@ import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -62,17 +61,16 @@ public static SdmxWebManager of(@Nonnull Iterable drive drivers.forEach(driverList::add); ConcurrentMap entryPointByName = new ConcurrentHashMap<>(); - ConcurrentMap cache = new ConcurrentHashMap(); - updateEntryPointMap(entryPointByName, driverList.stream().flatMap(o -> tryGetDefaultEntryPoints(o).stream())); - updateCache(driverList, cache); - return new SdmxWebManager(driverList, entryPointByName, new AtomicReference<>(cache)); + HasCache cacheSupport = HasCache.of(ConcurrentHashMap::new, (o, n) -> applyCache(n, driverList)); + + return new SdmxWebManager(driverList, entryPointByName, cacheSupport); } private final List drivers; private final ConcurrentMap entryPointByName; - private final AtomicReference cache; + private final HasCache cacheSupport; @Override public SdmxConnection getConnection(String name, LanguagePriorityList languages) throws IOException { @@ -101,13 +99,12 @@ public SdmxConnection getConnection(@Nonnull SdmxWebEntryPoint entryPoint, @Nonn @Override public ConcurrentMap getCache() { - return cache.get(); + return cacheSupport.getCache(); } @Override public void setCache(ConcurrentMap cache) { - this.cache.set(cache != null ? cache : new ConcurrentHashMap()); - updateCache(drivers, this.cache.get()); + this.cacheSupport.setCache(cache); } @Nonnull @@ -119,7 +116,7 @@ public void setEntryPoints(@Nonnull List list) { updateEntryPointMap(entryPointByName, list.stream()); } - private static void updateCache(List drivers, ConcurrentMap cache) { + private static void applyCache(ConcurrentMap cache, List drivers) { drivers.stream() .filter(HasCache.class::isInstance) .forEach(o -> ((HasCache) o).setCache(cache)); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java index 0b32c4bb..dce50b2d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java @@ -23,6 +23,7 @@ import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CONNECT_TIMEOUT; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.READ_TIMEOUT; import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import java.io.IOException; @@ -30,12 +31,11 @@ import java.net.URISyntaxException; import java.time.Clock; import java.util.Collection; -import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; @@ -43,68 +43,63 @@ * * @author Philippe Charles */ +@lombok.Builder(builderClassName = "Builder") @ThreadSafe -public final class ConnectorsDriverSupport implements HasCache { +public final class ConnectorsDriverSupport implements SdmxWebDriver, HasCache { - @Nonnull - public static ConnectorsDriverSupport of(@Nonnull String prefix, @Nonnull Class clazz) { - return new ConnectorsDriverSupport(prefix, GenericSDMXClientSupplier.ofType(clazz), new ConcurrentHashMap(), Clock.systemDefaultZone()); - } + @lombok.NonNull + private final String prefix; - @Nonnull - public static ConnectorsDriverSupport of(@Nonnull String prefix, @Nonnull GenericSDMXClientSupplier supplier) { - return new ConnectorsDriverSupport(prefix, supplier, new ConcurrentHashMap(), Clock.systemDefaultZone()); - } + @lombok.NonNull + private final BiFunction, GenericSDMXClient> supplier; - private final String prefix; - private final GenericSDMXClientSupplier supplier; - private final AtomicReference cache; - private final Clock clock; - - private ConnectorsDriverSupport(String prefix, GenericSDMXClientSupplier supplier, ConcurrentMap cache, Clock clock) { - this.prefix = prefix; - this.supplier = supplier; - this.cache = new AtomicReference<>(cache); - this.clock = clock; - } + @lombok.Singular + private final Collection entryPoints; + + @lombok.Builder.Default + private final Clock clock = Clock.systemDefaultZone(); + + @lombok.Builder.Default + private final HasCache cacheSupport = HasCache.of(ConcurrentHashMap::new); + @Override public SdmxConnection connect(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { return new ConnectorsConnection(getResource(entryPoint, languages)); } + @Override public boolean accepts(SdmxWebEntryPoint entryPoint) throws IOException { return entryPoint.getUri().toString().startsWith(prefix); } + @Override + public Collection getDefaultEntryPoints() { + return entryPoints; + } + @Override public ConcurrentMap getCache() { - return cache.get(); + return cacheSupport.getCache(); } @Override public void setCache(ConcurrentMap cache) { - this.cache.set(cache != null ? cache : new ConcurrentHashMap()); + cacheSupport.setCache(cache); } - @Nonnull - public static Collection entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { - return Collections.singleton(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); + private ConnectorsConnection.Resource getResource(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + return getResource(getEndpoint(entryPoint, prefix), entryPoint.getProperties(), languages); } - private ConnectorsConnection.Resource getResource(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { - Map info = entryPoint.getProperties(); - try { - URI endpoint = new URI(entryPoint.getUri().toString().substring(prefix.length())); - GenericSDMXClient client = supplier.getClient(endpoint, info, languages); - applyTimeouts(client, info); - return CachedResource.of(client, endpoint, languages, cache.get(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); - } catch (URISyntaxException ex) { - throw new IOException(ex); - } + private ConnectorsConnection.Resource getResource(URI endpoint, Map info, LanguagePriorityList languages) throws IOException { + GenericSDMXClient client = supplier.apply(endpoint, info); + configure(client, info, languages); + return CachedResource.of(client, endpoint, languages, getCache(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); } - private static void applyTimeouts(GenericSDMXClient client, Map info) { + private static void configure(GenericSDMXClient client, Map info, LanguagePriorityList languages) { if (client instanceof RestSdmxClient) { + ((RestSdmxClient) client).setLanguages(Util.fromLanguages(languages)); ((RestSdmxClient) client).setConnectTimeout(CONNECT_TIMEOUT.get(info, DEFAULT_CONNECT_TIMEOUT)); ((RestSdmxClient) client).setReadTimeout(READ_TIMEOUT.get(info, DEFAULT_READ_TIMEOUT)); } @@ -113,4 +108,47 @@ private static void applyTimeouts(GenericSDMXClient client, Map info) { private final static int DEFAULT_CONNECT_TIMEOUT = 1000 * 60 * 2; // 2 minutes private final static int DEFAULT_READ_TIMEOUT = 1000 * 60 * 2; // 2 minutes private static final long DEFAULT_CACHE_TTL = TimeUnit.MINUTES.toMillis(5); + + @Nonnull + public static URI getEndpoint(@Nonnull SdmxWebEntryPoint o, @Nonnull String prefix) throws IOException { + try { + return new URI(o.getUri().toString().substring(prefix.length())); + } catch (URISyntaxException ex) { + throw new IOException(ex); + } + } + + @FunctionalInterface + public interface ClientConstructor { + + @Nonnull + GenericSDMXClient get() throws URISyntaxException; + } + + public static final class Builder { + + public Builder supplier(@Nonnull ClientConstructor constructor) { + this.supplier = (x, y) -> newClient(constructor, x); + return this; + } + + public Builder supplier(@Nonnull BiFunction, GenericSDMXClient> supplier) { + this.supplier = supplier; + return this; + } + + public Builder entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { + return entryPoint(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); + } + + private static GenericSDMXClient newClient(ClientConstructor constructor, URI endpoint) { + try { + GenericSDMXClient result = constructor.get(); + result.setEndpoint(endpoint); + return result; + } catch (URISyntaxException ex) { + throw new RuntimeException(ex); + } + } + } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java deleted file mode 100644 index fd44d316..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientSupplier.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.LanguagePriorityList; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import it.bancaditalia.oss.sdmx.client.RestSdmxClient; -import java.net.URI; -import java.util.Map; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -public interface GenericSDMXClientSupplier { - - @Nonnull - GenericSDMXClient getClient(@Nonnull URI endpoint, @Nonnull Map info, @Nonnull LanguagePriorityList langs); - - @Nonnull - static GenericSDMXClientSupplier ofType(@Nonnull Class clazz) { - return (URI endpoint, Map info, LanguagePriorityList langs) -> { - try { - RestSdmxClient result = clazz.newInstance(); - result.setEndpoint(endpoint); - result.setLanguages(Util.fromLanguages(langs)); - return result; - } catch (InstantiationException | IllegalAccessException ex) { - throw new RuntimeException(ex); - } - }; - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index f1225ee3..db1b3098 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -34,7 +34,7 @@ * * @author Philippe Charles */ -final class PortableTimeSeriesCursor implements DataCursor { +public final class PortableTimeSeriesCursor implements DataCursor { private final Iterator data; private final ObsParser obs; @@ -43,7 +43,7 @@ final class PortableTimeSeriesCursor implements DataCursor { private boolean closed; private boolean hasObs; - PortableTimeSeriesCursor(List data, ObsParser obs) { + public PortableTimeSeriesCursor(List data, ObsParser obs) { this.data = data.iterator(); this.obs = obs; this.closed = false; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 26af185f..c548b274 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -69,7 +69,7 @@ public it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.s } public boolean isNoResultMatchingQuery(SdmxException ex) { - return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == 100; + return ex instanceof SdmxResponseException && ((SdmxResponseException) ex).getResponseCode() == SdmxResponseException.SDMX_NO_RESULTS_FOUND; } public static final BoolProperty SUPPORTS_COMPRESSION = new BoolProperty("supportsCompression"); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java index dfdfb91d..319087dc 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ABS; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class AbsDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:abs:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, ABS.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:abs:") + .supplier(ABS::new) + .entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java index 3dd99cb6..ebe656aa 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.EUROSTAT; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class EurostatDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:eurostat:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, EUROSTAT.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:eurostat:") + .supplier(EUROSTAT::new) + .entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java index 310d4979..cc325adc 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ILO; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class IloDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:ilo:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, ILO.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:ilo:") + .supplier(ILO::new) + .entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java index 22e81b08..dda79023 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.IMF; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class ImfDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:imf:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, IMF.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:imf:") + .supplier(IMF::new) + .entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 69b06b97..e280f499 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -19,7 +19,6 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; @@ -34,7 +33,6 @@ import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; import java.io.IOException; -import java.util.Collection; import java.util.List; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; @@ -59,25 +57,29 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class InseeDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:insee:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new InseeClient(u, l)); + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:insee:") + .supplier((u, i) -> new InseeClient(u)) + .entryPoint(INSEE_ENTRY) + .build(); @SdmxFix(id = "INSEE#1", cause = "Fallback to http due to some servers that use root certificate unknown to jdk'") - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("INSEE", "Institut national de la statistique et des études économiques", "sdmx:insee:http://bdm.insee.fr/series/sdmx"); - } + private static final SdmxWebEntryPoint INSEE_ENTRY = SdmxWebEntryPoint + .builder() + .name("INSEE") + .description("Institut national de la statistique et des études économiques") + .uri("sdmx:insee:http://bdm.insee.fr/series/sdmx") + .build(); private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { @SdmxFix(id = "INSEE#2", cause = "Does not follow sdmx standard codes") private final SdmxDialect dialect; - private InseeClient(URI endpoint, LanguagePriorityList langs) { + private InseeClient(URI endpoint) { super("", endpoint, false, false, true); - this.languages = Util.fromLanguages(langs); this.dialect = new InseeDialect(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java index 41910fd8..b8db52ee 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.NBB; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class NbbDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:nbb:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, NBB.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:nbb:") + .supplier(NBB::new) + .entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java index 81f3b541..353f2ce7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.OECD; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class OecdDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:oecd:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, OECD.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:oecd:") + .supplier(OECD::new) + .entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java index e1f60b5e..e58ed9f3 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java @@ -16,18 +16,13 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.LanguagePriorityList; import static internal.connectors.Util.NEEDS_CREDENTIALS; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.RestSdmx20Client; -import java.util.Collections; -import java.util.List; import java.util.Map; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; -import internal.connectors.Util; import java.net.URI; /** @@ -37,21 +32,17 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class Sdmx20Driver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:sdmx20:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, Sdmx20Client::new); - - @Override - public List getDefaultEntryPoints() { - return Collections.emptyList(); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:sdmx20:") + .supplier(Sdmx20Client::new) + .build(); private static final class Sdmx20Client extends RestSdmx20Client { - private Sdmx20Client(URI endpoint, Map info, LanguagePriorityList langs) { + private Sdmx20Client(URI endpoint, Map info) { super("", endpoint, NEEDS_CREDENTIALS.get(info, false), null, "compact_v2"); - this.languages = Util.fromLanguages(langs); } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index c20d54ef..c23b53e2 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -18,7 +18,6 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.Series; @@ -55,10 +54,14 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx21:"; @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, (u, i, l) -> new Sdmx21Client(u, Sdmx21Config.load(i), l)); - - @Override - public List getDefaultEntryPoints() { + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix(PREFIX) + .supplier((u, i) -> new Sdmx21Client(u, Sdmx21Config.load(i))) + .entryPoints(getEntryPoints()) + .build(); + + private static List getEntryPoints() { Sdmx21EntryPointBuilder b = new Sdmx21EntryPointBuilder(); List result = new ArrayList<>(); result.add(b.clear() @@ -103,7 +106,7 @@ public List getDefaultEntryPoints() { .description("World Bank") .endpoint("https://api.worldbank.org/v2/sdmx/rest") .supportsCompression(true) -// .seriesKeysOnlySupported(true) + // .seriesKeysOnlySupported(true) .build()); return result; } @@ -179,9 +182,8 @@ private final static class Sdmx21Client extends RestSdmxClient implements HasDat private final Sdmx21Config config; - private Sdmx21Client(URI endpoint, Sdmx21Config config, LanguagePriorityList langs) { + private Sdmx21Client(URI endpoint, Sdmx21Config config) { super("", endpoint, config.isNeedsCredentials(), config.isNeedsURLEncoding(), config.isSupportsCompression()); - this.languages = Util.fromLanguages(langs); this.config = config; } @@ -205,7 +207,8 @@ private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resou private Parser> getCompactData21Parser(DataFlowStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).parse(new XMLEventStreamReader(r), () -> {})) { + try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).parse(new XMLEventStreamReader(r), () -> { + })) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { throw new SdmxIOException("Cannot parse compact data 21", ex); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java index f4bff6b9..dcf4a086 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java @@ -16,10 +16,8 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.UIS; -import java.util.Collection; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorsDriverSupport; @@ -31,13 +29,11 @@ @ServiceProvider(service = SdmxWebDriver.class) public final class UisDriver implements SdmxWebDriver, HasCache { - private static final String PREFIX = "sdmx:uis:"; - @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport.of(PREFIX, UIS.class); - - @Override - public Collection getDefaultEntryPoints() { - return ConnectorsDriverSupport.entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx"); - } + private final ConnectorsDriverSupport support = ConnectorsDriverSupport + .builder() + .prefix("sdmx:uis:") + .supplier(UIS::new) + .entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx") + .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java index d299c6b0..e552beea 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java @@ -73,6 +73,7 @@ private static Frequency parseInseeFreq(String code) { case 'M': return MONTHLY; case 'B': + // FIXME: define new freq? return MONTHLY; } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java index 1e7b5616..efff6d03 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java @@ -17,6 +17,7 @@ package be.nbb.sdmx.facade.web; import be.nbb.sdmx.facade.LanguagePriorityList; +import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; @@ -25,7 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -42,17 +43,17 @@ public void testCompliance() { @Test @SuppressWarnings("null") public void testFactories() { - assertThatThrownBy(() -> SdmxWebManager.of((Iterable) null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> SdmxWebManager.of((SdmxWebDriver[]) null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((Iterable) null)); + assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((SdmxWebDriver[]) null)); } @Test @SuppressWarnings("null") public void testGetConnectionOfEntryPoint() { SdmxWebManager manager = SdmxWebManager.of(REPO); - assertThatThrownBy(() -> manager.getConnection((SdmxWebEntryPoint) null, LanguagePriorityList.ANY)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> manager.getConnection(HELLO, null)).isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> manager.getConnection(HELLO.toBuilder().uri("ko").build(), LanguagePriorityList.ANY)).isInstanceOf(IOException.class); + assertThatNullPointerException().isThrownBy(() -> manager.getConnection((SdmxWebEntryPoint) null, ANY)); + assertThatNullPointerException().isThrownBy(() -> manager.getConnection(HELLO, null)); + assertThatIOException().isThrownBy(() -> manager.getConnection(HELLO.toBuilder().uri("ko").build(), ANY)); } private static final SdmxWebEntryPoint HELLO = SdmxWebEntryPoint.builder().name("ok").uri(RepoDriver.PREFIX + "r1").build(); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java index aa244093..5fc9734d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java @@ -36,7 +36,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatNullPointerException; import org.junit.Test; /** @@ -80,7 +80,7 @@ public void testLoadDataflow() throws IOException { CachedResource target = new CachedResource(resource, "", cache, clock, 100); - assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); target.loadDataflow(sample); assertThat(resource.count).isEqualTo(1); @@ -115,7 +115,7 @@ public void testLoadDataStructure() throws IOException { CachedResource target = new CachedResource(resource, "", cache, clock, 100); - assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); target.loadDataStructure(sample); assertThat(resource.count).isEqualTo(2); @@ -144,7 +144,7 @@ public void testLoadData() throws IOException { CachedResource target = new CachedResource(resource, "", cache, clock, 100); - assertThatThrownBy(() -> target.loadDataflow(null)).isInstanceOf(NullPointerException.class); + assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); target.loadData(sample, Key.ALL, true); assertThat(resource.count).isEqualTo(3); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java new file mode 100644 index 00000000..c172bb89 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class ConnectorsDriverSupportTest { + + @Test + public void testFactories() { + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index fd604409..6b9c191c 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -16,6 +16,7 @@ */ package internal.connectors; +import test.ConnectorsResource; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.samples.SdmxSource; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java new file mode 100644 index 00000000..b53a766e --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import it.bancaditalia.oss.sdmx.exceptions.SdmxExceptionFactory; +import java.net.HttpURLConnection; +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class UtilTest { + + @Test + public void testIsNoResultMatchingQuery() { + assertThat(Util.isNoResultMatchingQuery(SdmxExceptionFactory.createRestException(HttpURLConnection.HTTP_NOT_FOUND, null, null))).isTrue(); + assertThat(Util.isNoResultMatchingQuery(SdmxExceptionFactory.createRestException(HttpURLConnection.HTTP_BAD_REQUEST, null, null))).isFalse(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java new file mode 100644 index 00000000..ae4a5a39 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class AbsDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new AbsDriver(), "sdmx:abs:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java new file mode 100644 index 00000000..065e33c1 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class EurostatDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new EurostatDriver(), "sdmx:eurostat:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java new file mode 100644 index 00000000..665ccddb --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class IloDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new IloDriver(), "sdmx:ilo:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java new file mode 100644 index 00000000..b47da849 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class ImfDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new ImfDriver(), "sdmx:imf:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java new file mode 100644 index 00000000..45570a1d --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class InseeDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new InseeDriver(), "sdmx:insee:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java new file mode 100644 index 00000000..8c9d3a10 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class NbbDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new NbbDriver(), "sdmx:nbb:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java new file mode 100644 index 00000000..d81ee3f3 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class OecdDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new OecdDriver(), "sdmx:oecd:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java new file mode 100644 index 00000000..26b73114 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import java.io.IOException; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class Sdmx20DriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new Sdmx20Driver(), "sdmx:sdmx20:"); + } + + @Test + public void testConnect() throws IOException { + SdmxWebEntryPoint x = SdmxWebEntryPoint.builder().name("").uri("sdmx:sdmx20:http://localhost").build(); + assertThatCode(() -> new Sdmx20Driver().connect(x, ANY).close()).doesNotThrowAnyException(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java new file mode 100644 index 00000000..08236480 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class Sdmx21DriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new Sdmx21Driver(), "sdmx:sdmx21:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java new file mode 100644 index 00000000..8e1ce41d --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors.drivers; + +import org.junit.Test; +import static test.DriverAssertions.*; + +/** + * + * @author Philippe Charles + */ +public class UisDriverTest { + + @Test + public void testCompliance() { + assertDriverCompliance(new UisDriver(), "sdmx:uis:"); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java new file mode 100644 index 00000000..0db80a97 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util.drivers; + +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dimension; +import static be.nbb.sdmx.facade.Frequency.*; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.Chars; +import java.util.function.Function; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; +import test.DialectAssertions; + +/** + * + * @author Philippe Charles + */ +public class InseeDialectTest { + + @Test + public void testCompliance() { + DialectAssertions.assertDialectCompliance(new InseeDialect()); + } + + @Test + public void testFreqParser() { + InseeDialect d = new InseeDialect(); + DataStructure dsd = DataStructure.builder() + .dimension(Dimension.builder().id("FREQ").position(1).label("").build()) + .ref(DataStructureRef.parse("abc")) + .label("") + .build(); + Key.Builder key = Key.builder(dsd); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "A"), Function.identity())).isEqualTo(ANNUAL); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "T"), Function.identity())).isEqualTo(QUARTERLY); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "M"), Function.identity())).isEqualTo(MONTHLY); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "B"), Function.identity())).isEqualTo(MONTHLY); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "S"), Function.identity())).isEqualTo(HALF_YEARLY); + assertThat(d.getFreqParser(dsd).parse(key.put("FREQ", "X"), Function.identity())).isEqualTo(UNDEFINED); + } + + @Test + public void testPeriodParser() { + InseeDialect d = new InseeDialect(); + assertThat(d.getPeriodParser(ANNUAL).parse("2013")).isEqualTo("2013-01-01T00:00:00"); + assertThat(d.getPeriodParser(QUARTERLY).parse("2014-Q3")).isEqualTo("2014-07-01T00:00:00"); + assertThat(d.getPeriodParser(MONTHLY).parse("1990-09")).isEqualTo("1990-09-01T00:00:00"); + assertThat(d.getPeriodParser(HALF_YEARLY).parse("2012-S2")).isEqualTo("2012-07-01T00:00:00"); + assertThat(d.getPeriodParser(MINUTELY).parse("2012-S2")).isNull(); + } + + @Test + @SuppressWarnings("null") + public void testValueParser() { + Chars.Parser p = new InseeDialect().getValueParser(); + assertThat(p.parse("hello")).isNull(); + assertThat(p.parse("3.14")).isEqualTo(3.14); + assertThatNullPointerException().isThrownBy(() -> p.parse(null)); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/CacheAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/CacheAssertions.java new file mode 100644 index 00000000..08370276 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/CacheAssertions.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test; + +import be.nbb.sdmx.facade.util.HasCache; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import static org.assertj.core.api.Assertions.*; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class CacheAssertions { + + public void assertCacheBehavior(HasCache o) { + assertThat(o.getCache()).isNotNull(); + ConcurrentMap cache = new ConcurrentHashMap(); + o.setCache(cache); + assertThat(o.getCache()).isSameAs(cache); + o.setCache(null); + assertThat(o.getCache()).isNotNull().isNotSameAs(cache); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java similarity index 93% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java index 2311ce26..6854ede5 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package test; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Frequency; @@ -25,6 +25,8 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.ObsParser; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.connectors.PortableTimeSeriesCursor; +import internal.connectors.Util; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; @@ -110,7 +112,7 @@ List data20(ByteSource xml, DataFlowStructure dsd, LanguageP .collect(Collectors.toList()); } - List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { + public List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser()); } @@ -118,7 +120,7 @@ List flow21(ByteSource xml, LanguagePriorityList l) throws IOException return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()); } - List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { + public List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl return FacadeResource.data21(XIF, xml, Util.toStructure(dsd)) .stream() diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DialectAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DialectAssertions.java new file mode 100644 index 00000000..65b5ab30 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DialectAssertions.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test; + +import be.nbb.sdmx.facade.parser.spi.SdmxDialect; +import static org.assertj.core.api.Assertions.*; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class DialectAssertions { + + @SuppressWarnings("null") + public void assertDialectCompliance(SdmxDialect d) { + assertThat(d.getName()).isNotBlank(); + assertThat(d.getDescription()).isNotBlank(); + assertThatNullPointerException().isThrownBy(() -> d.getFreqParser(null)); + assertThatNullPointerException().isThrownBy(() -> d.getPeriodParser(null)); + assertThat(d.getValueParser()).isNotNull(); + + assertThat(d.getClass()).isFinal(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java new file mode 100644 index 00000000..18ad78fe --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.util.IO; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorsDriverSupport; +import static org.assertj.core.api.Assertions.*; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class DriverAssertions { + + @SuppressWarnings("null") + public void assertDriverCompliance(SdmxWebDriver d, String prefix) { + if (d instanceof HasCache) { + CacheAssertions.assertCacheBehavior((HasCache) d); + } + + assertThatNullPointerException().isThrownBy(() -> d.accepts(null)); + assertThatNullPointerException().isThrownBy(() -> d.connect(null, LanguagePriorityList.ANY)); + assertThatNullPointerException().isThrownBy(() -> d.connect(SdmxWebEntryPoint.builder().name("").build(), null)); + + assertThat(d.getDefaultEntryPoints()) + .allSatisfy(o -> checkEntryPoint(o, prefix)) + .allMatch(IO.asUnchecked(d::accepts)); + + assertThat(d.getClass()).isFinal(); + } + + private void checkEntryPoint(SdmxWebEntryPoint o, String prefix) { + assertThat(o.getName()).isNotBlank(); + assertThat(o.getDescription()).isNotBlank(); + assertThat(o.getProperties()).isNotNull(); + assertThat(o.getUri().toString()).startsWith(prefix); + assertThatCode(() -> ConnectorsDriverSupport.getEndpoint(o, prefix).toURL()).doesNotThrowAnyException(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java similarity index 99% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java index 45a38b76..7da69bc9 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package test; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -28,6 +28,7 @@ import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.sdmx.facade.xml.stream.Stax; +import internal.connectors.Util; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java index c77a8646..dcd65693 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ParsersTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package test; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; From 2e5126dbee8beff18924b6557a09829e7d027f36 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 23 Nov 2017 15:33:54 +0100 Subject: [PATCH 52/68] Added REST layer to separate concerns. --- .../nbb/sdmx/facade/repo/SdmxRepository.java | 83 ++++++--- .../internal/connectors/CachedResource.java | 158 ---------------- .../connectors/ConnectorRestClient.java | 173 ++++++++++++++++++ .../connectors/GenericSDMXClientResource.java | 155 ---------------- .../internal/connectors/HasDataCursor.java | 6 +- .../main/java/internal/connectors/Util.java | 46 +++++ .../connectors/drivers/AbsDriver.java | 9 +- .../connectors/drivers/EurostatDriver.java | 9 +- .../connectors/drivers/IloDriver.java | 9 +- .../connectors/drivers/ImfDriver.java | 9 +- .../connectors/drivers/InseeDriver.java | 35 ++-- .../connectors/drivers/NbbDriver.java | 9 +- .../connectors/drivers/OecdDriver.java | 9 +- .../connectors/drivers/Sdmx20Driver.java | 7 +- .../connectors/drivers/Sdmx21Driver.java | 27 +-- .../connectors/drivers/UisDriver.java | 9 +- .../java/internal/web/CachedRestClient.java | 162 ++++++++++++++++ .../java/internal/web/FailsafeRestClient.java | 128 +++++++++++++ .../main/java/internal/web/RestClient.java | 60 ++++++ .../RestConnection.java} | 76 ++++---- .../RestDriverSupport.java} | 72 ++------ .../ConnectorsDriverSupportTest.java | 30 --- .../java/internal/connectors/UtilTest.java | 45 ++++- .../CachedRestClientTest.java} | 160 +++++++--------- .../RestConnectionTest.java} | 7 +- .../src/test/java/test/DriverAssertions.java | 4 +- .../src/test/java/test/NoOpRestClient.java | 74 ++++++++ .../src/test/java/test/RepoRestClient.java | 69 +++++++ 28 files changed, 1009 insertions(+), 631 deletions(-) delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java delete mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java rename sdmx-facade/sdmx-facade-web/src/main/java/internal/{connectors/ConnectorsConnection.java => web/RestConnection.java} (63%) rename sdmx-facade/sdmx-facade-web/src/main/java/internal/{connectors/ConnectorsDriverSupport.java => web/RestDriverSupport.java} (52%) delete mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java rename sdmx-facade/sdmx-facade-web/src/test/java/internal/{connectors/CachedResourceTest.java => web/CachedRestClientTest.java} (50%) rename sdmx-facade/sdmx-facade-web/src/test/java/internal/{connectors/ConnectorsConnectionTest.java => web/RestConnectionTest.java} (83%) create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 90ca5379..55ac526f 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -22,6 +22,7 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; @@ -30,10 +31,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Stream; import javax.annotation.Nonnull; -import javax.annotation.Nullable; /** * @@ -65,20 +67,50 @@ public SdmxConnection asConnection() { return new RepoConnection(this); } - @Nullable - public DataCursor getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { + @Nonnull + public Optional getStructure(@Nonnull DataStructureRef ref) { + Objects.requireNonNull(ref); + return dataStructures + .stream() + .filter(o -> o.getRef().equals(ref)) + .findFirst(); + } + + @Nonnull + public Optional getFlow(@Nonnull DataflowRef ref) { + Objects.requireNonNull(ref); + return dataflows + .stream() + .filter(o -> o.getRef().equals(ref)) + .findFirst(); + } + + @Nonnull + public Optional getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { + return getData(flowRef).map(toCursor(query)); + } + + @Nonnull + public Optional> getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { + return getData(flowRef).map(toStream(query)); + } + + @Nonnull + private Optional> getData(@Nonnull DataflowRef flowRef) { Objects.requireNonNull(flowRef); + return Optional.ofNullable(data.get(flowRef)); + } + + @Nonnull + private static Function, DataCursor> toCursor(@Nonnull DataQuery query) { Objects.requireNonNull(query); - List result = data.get(flowRef); - return result != null ? SeriesSupport.asCursor(result, query.getKey()) : null; + return o -> SeriesSupport.asCursor(o, query.getKey()); } - @Nullable - public Stream getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { - Objects.requireNonNull(flowRef); + @Nonnull + private static Function, Stream> toStream(@Nonnull DataQuery query) { Objects.requireNonNull(query); - List result = data.get(flowRef); - return result != null ? result.stream().filter(o -> query.getKey().contains(o.getKey())) : null; + return o -> o.stream().filter(s -> query.getKey().contains(s.getKey())); } public static final class Builder { @@ -129,42 +161,33 @@ public Set getFlows() throws IOException { @Override public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); - Objects.requireNonNull(flowRef); - return repo.getDataflows() - .stream() - .filter(o -> flowRef.equals(o.getRef())) - .findFirst() + return repo + .getFlow(flowRef) .orElseThrow(() -> new IOException("Dataflow not found")); } @Override public DataStructure getStructure(DataflowRef flowRef) throws IOException { - Dataflow flow = getFlow(flowRef); - return repo.getDataStructures() - .stream() - .filter(o -> flow.getStructureRef().equals(o.getRef())) - .findFirst() + checkState(); + return repo + .getStructure(getFlow(flowRef).getStructureRef()) .orElseThrow(() -> new IOException("DataStructure not found")); } @Override public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - DataCursor result = repo.getCursor(flowRef, query); - if (result != null) { - return result; - } - throw new IOException("Data not found"); + return repo + .getCursor(flowRef, query) + .orElseThrow(() -> new IOException("Data not found")); } @Override public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - Stream result = repo.getStream(flowRef, query); - if (result != null) { - return result; - } - throw new IOException("Data not found"); + return repo + .getStream(flowRef, query) + .orElseThrow(() -> new IOException("Data not found")); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java deleted file mode 100644 index abb25fc4..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/CachedResource.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.util.TtlCache; -import be.nbb.sdmx.facade.util.TypedId; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import java.io.IOException; -import java.net.URI; -import java.time.Clock; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentMap; - -/** - * - * @author Philippe Charles - */ -final class CachedResource extends GenericSDMXClientResource { - - static CachedResource of(GenericSDMXClient client, URI endpoint, LanguagePriorityList languages, ConcurrentMap cache, Clock clock, long ttlInMillis) { - return new CachedResource(client, generateBase(endpoint.getHost(), languages), cache, clock, ttlInMillis); - } - - private final TtlCache cache; - private final TypedId> idOfFlows; - private final TypedId idOfFlow; - private final TypedId idOfStruct; - private final TypedId idOfKeysOnly; - - CachedResource(GenericSDMXClient client, String base, ConcurrentMap cache, Clock clock, long ttlInMillis) { - super(client); - this.cache = TtlCache.of(cache, clock, ttlInMillis); - this.idOfFlows = TypedId.of("flows://" + base); - this.idOfFlow = TypedId.of("flow://" + base); - this.idOfStruct = TypedId.of("struct://" + base); - this.idOfKeysOnly = TypedId.of("keys://" + base); - } - - @Override - public Map loadDataFlowsById() throws IOException { - return loadDataFlowsWithCache(); - } - - @Override - public Dataflow loadDataflow(DataflowRef flowRef) throws IOException { - Objects.requireNonNull(flowRef); - - Dataflow result = peekDataflowFromCache(flowRef); - return result != null ? result : loadDataflowWithCache(flowRef); - } - - @Override - public DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { - return loadDataStructureWithCache(flowRef); - } - - @Override - public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - if (!serieskeysonly) { - return super.loadData(flowRef, key, serieskeysonly); - } - DataCursor result = loadKeysOnlyWithCache(flowRef, key).getCursor(flowRef, DataQuery.of(key, true)); - if (result != null) { - return result; - } - throw new IOException("Data not found"); - } - - @Override - public boolean isSeriesKeysOnlySupported() { - return super.isSeriesKeysOnlySupported(); - } - - private Map loadDataFlowsWithCache() throws IOException { - Map result = cache.get(idOfFlows); - if (result == null) { - result = super.loadDataFlowsById(); - cache.put(idOfFlows, result); - } - return result; - } - - private DataFlowStructure loadDataStructureWithCache(DataflowRef flowRef) throws IOException { - TypedId id = idOfStruct.with(flowRef); - DataFlowStructure result = cache.get(id); - if (result == null) { - result = super.loadDataStructure(flowRef); - cache.put(id, result); - } - return result; - } - - private SdmxRepository loadKeysOnlyWithCache(DataflowRef flowRef, Key key) throws IOException { - TypedId id = idOfKeysOnly.with(flowRef); - SdmxRepository result = cache.get(id); - if (result == null || isBroaderRequest(key, result)) { - result = copyDataKeys(flowRef, key); - cache.put(id, result); - } - return result; - } - - private Dataflow peekDataflowFromCache(DataflowRef flowRef) { - // check if dataflow has been already loaded by #loadDataFlowsById - Map dataFlows = cache.get(idOfFlows); - return dataFlows != null ? dataFlows.get(flowRef.getId()) : null; - } - - private Dataflow loadDataflowWithCache(DataflowRef flowRef) throws IOException { - TypedId id = idOfFlow.with(flowRef); - Dataflow result = cache.get(id); - if (result == null) { - result = super.loadDataflow(flowRef); - cache.put(id, result); - } - return result; - } - - private boolean isBroaderRequest(Key key, SdmxRepository repo) { - return key.supersedes(Key.parse(repo.getName())); - } - - private SdmxRepository copyDataKeys(DataflowRef flowRef, Key key) throws IOException { - try (DataCursor cursor = super.loadData(flowRef, key, true)) { - return SdmxRepository.builder() - .copyOf(flowRef, cursor) - .name(key.toString()) - .build(); - } - } - - private static String generateBase(String host, LanguagePriorityList languages) { - return host + languages.toString() + "/"; - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java new file mode 100644 index 00000000..77a9ae49 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java @@ -0,0 +1,173 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.parser.ObsParser; +import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CONNECT_TIMEOUT; +import static be.nbb.sdmx.facade.util.CommonSdmxProperty.READ_TIMEOUT; +import be.nbb.sdmx.facade.util.NoOpCursor; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import java.io.IOException; +import java.util.List; +import internal.web.RestClient; +import it.bancaditalia.oss.sdmx.client.RestSdmxClient; +import it.bancaditalia.oss.sdmx.client.custom.DotStat; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.AccessLevel; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor(access = AccessLevel.PRIVATE) +public final class ConnectorRestClient implements RestClient { + + @FunctionalInterface + public interface ConnectorConstructor { + + @Nonnull + RestSdmxClient get() throws URISyntaxException; + } + + public static RestClient.Supplier of(ConnectorConstructor supplier) { + return (x, prefix, z) -> { + try { + RestSdmxClient client = supplier.get(); + client.setEndpoint(getEndpoint(x, prefix)); + configure(client, x.getProperties(), z); + return new ConnectorRestClient(client); + } catch (URISyntaxException ex) { + throw new RuntimeException(ex); + } + }; + } + + public static RestClient.Supplier of(BiFunction, RestSdmxClient> supplier) { + return (x, prefix, z) -> { + try { + RestSdmxClient client = supplier.apply(getEndpoint(x, prefix), x.getProperties()); + configure(client, x.getProperties(), z); + return new ConnectorRestClient(client); + } catch (URISyntaxException ex) { + throw new RuntimeException(ex); + } + }; + } + + private final RestSdmxClient connector; + + @Override + public List getFlows() throws IOException { + try { + return connector + .getDataflows() + .values() + .stream() + .map(Util::toFlow) + .collect(Collectors.toList()); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get datasets"); + } + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + try { + return Util.toFlow(connector.getDataflow(ref.getId(), ref.getAgency(), ref.getVersion())); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get details from dataset '%s'", ref); + } + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + try { + return Util.toStructure(connector.getDataFlowStructure(Util.fromStructureRef(ref), true)); + } catch (SdmxException ex) { + throw expected(ex, "Failed to get data structure from dataset '%s'", ref); + } + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + try { + return connector instanceof HasDataCursor + ? getCursor((HasDataCursor) connector, flowRef, dsd, query) + : getAdaptedCursor(connector, flowRef, dsd, query); + } catch (SdmxException ex) { + if (Util.isNoResultMatchingQuery(ex)) { + return NoOpCursor.noOp(); + } + throw expected(ex, "Failed to get data from dataset '%s' with key '%s'", flowRef, query.getKey()); + } + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return connector instanceof HasSeriesKeysOnlySupported + && ((HasSeriesKeysOnlySupported) connector).isSeriesKeysOnlySupported(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef ref) { + return connector instanceof DotStat ? DataStructureRef.of(ref.getAgency(), ref.getId(), ref.getVersion()) : null; + } + + private static DataCursor getCursor(HasDataCursor connector, DataflowRef flowRef, DataStructure dsd, DataQuery query) throws SdmxException, IOException { + return connector.getDataCursor(flowRef, dsd, query.getKey(), isSeriesKeyOnly(query)); + } + + private static DataCursor getAdaptedCursor(RestSdmxClient connector, DataflowRef flowRef, DataStructure dsd, DataQuery query) throws SdmxException { + return new PortableTimeSeriesCursor(connector.getTimeSeries(Util.fromFlowQuery(flowRef), Util.fromStructure(dsd), query.getKey().toString(), null, null, isSeriesKeyOnly(query), null, false), ObsParser.standard()); + } + + private static boolean isSeriesKeyOnly(DataQuery query) { + return query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); + } + + private static IOException expected(SdmxException ex, String format, Object... args) { + return new IOException(String.format(format, args), ex); + } + + private static void configure(RestSdmxClient client, Map info, LanguagePriorityList languages) { + client.setLanguages(Util.fromLanguages(languages)); + client.setConnectTimeout(CONNECT_TIMEOUT.get(info, DEFAULT_CONNECT_TIMEOUT)); + client.setReadTimeout(READ_TIMEOUT.get(info, DEFAULT_READ_TIMEOUT)); + } + + @Nonnull + private static URI getEndpoint(@Nonnull SdmxWebEntryPoint o, @Nonnull String prefix) throws URISyntaxException { + return new URI(o.getUri().toString().substring(prefix.length())); + } + + private final static int DEFAULT_CONNECT_TIMEOUT = 1000 * 60 * 2; // 2 minutes + private final static int DEFAULT_READ_TIMEOUT = 1000 * 60 * 2; // 2 minutes +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java deleted file mode 100644 index 0c92715b..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/GenericSDMXClientResource.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.NoOpCursor; -import be.nbb.sdmx.facade.parser.ObsParser; -import be.nbb.sdmx.facade.util.UnexpectedIOException; -import it.bancaditalia.oss.sdmx.api.DSDIdentifier; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import it.bancaditalia.oss.sdmx.client.custom.DotStat; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import java.io.IOException; -import java.util.Map; -import java.util.logging.Level; - -/** - * - * @author Philippe Charles - */ -@lombok.AllArgsConstructor -@lombok.extern.java.Log -class GenericSDMXClientResource implements ConnectorsConnection.Resource { - - private final GenericSDMXClient client; - - @Override - public Map loadDataFlowsById() throws IOException { - Map result; - - try { - result = client.getDataflows(); - } catch (SdmxException ex) { - throw expected(ex, "Failed to get datasets"); - } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting datasets"); - } - - if (result == null) { - throw unexpectedNull("Unexpected null while getting datsets"); - } - - return result; - } - - @Override - public it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(DataflowRef flowRef) throws IOException { - it.bancaditalia.oss.sdmx.api.Dataflow result; - - try { - result = client.getDataflow(flowRef.getId(), flowRef.getAgency(), flowRef.getVersion()); - } catch (SdmxException ex) { - throw expected(ex, "Failed to get details from dataset '%s'", flowRef); - } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting details from dataset '%s'", flowRef); - } - - if (result == null) { - throw unexpectedNull("Unexpected null while getting details from dataset '%s'", flowRef); - } - - return result; - } - - @Override - public it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(DataflowRef flowRef) throws IOException { - it.bancaditalia.oss.sdmx.api.DSDIdentifier dsd = loadDsdIdentifier(flowRef); - - it.bancaditalia.oss.sdmx.api.DataFlowStructure result; - - try { - result = client.getDataFlowStructure(dsd, true); - } catch (SdmxException ex) { - throw expected(ex, "Failed to get data structure from dataset '%s'", flowRef); - } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting data structure from dataset '%s'", flowRef); - } - - if (result == null) { - throw unexpectedNull("Unexpected null while getting data structure from dataset '%s'", flowRef); - } - - return result; - } - - @Override - public DataCursor loadData(DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { - it.bancaditalia.oss.sdmx.api.Dataflow dataflow = loadDataflow(flowRef); - it.bancaditalia.oss.sdmx.api.DataFlowStructure dfs = loadDataStructure(flowRef); - - DataCursor result; - - try { - result = client instanceof HasDataCursor - ? ((HasDataCursor) client).getDataCursor(dataflow, dfs, key, serieskeysonly) - : new PortableTimeSeriesCursor(client.getTimeSeries(dataflow, dfs, key.toString(), null, null, serieskeysonly, null, false), ObsParser.standard()); - } catch (SdmxException ex) { - if (Util.isNoResultMatchingQuery(ex)) { - return NoOpCursor.noOp(); - } - throw expected(ex, "Failed to get data from dataset '%s' with key '%s'", flowRef, key); - } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting data from dataset '%s' with key '%s'", flowRef, key); - } - - if (result == null) { - throw unexpectedNull("Unexpected null while getting data from dataset '%s' with key '%s'", flowRef, key); - } - - return result; - } - - @Override - public boolean isSeriesKeysOnlySupported() { - return client instanceof HasSeriesKeysOnlySupported - && ((HasSeriesKeysOnlySupported) client).isSeriesKeysOnlySupported(); - } - - private it.bancaditalia.oss.sdmx.api.DSDIdentifier loadDsdIdentifier(DataflowRef flowRef) throws IOException { - return client instanceof DotStat - ? new DSDIdentifier(flowRef.getId(), flowRef.getAgency(), flowRef.getVersion()) - : loadDataflow(flowRef).getDsdIdentifier(); - } - - private static IOException expected(SdmxException ex, String format, Object... args) { - return new IOException(String.format(format, args), ex); - } - - private static IOException unexpected(RuntimeException ex, String format, Object... args) { - log.log(Level.WARNING, format, args); - return new UnexpectedIOException(ex); - } - - private static IOException unexpectedNull(String format, Object... args) { - String msg = String.format(format, args); - log.log(Level.WARNING, msg); - return new UnexpectedIOException(new NullPointerException(msg)); - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java index 9aab6dc7..98f6638d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java @@ -17,9 +17,9 @@ package internal.connectors; import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import java.io.IOException; import javax.annotation.Nonnull; @@ -31,5 +31,5 @@ public interface HasDataCursor { @Nonnull - DataCursor getDataCursor(@Nonnull Dataflow dataflow, @Nonnull DataFlowStructure dsd, @Nonnull Key resource, boolean serieskeysonly) throws SdmxException, IOException; + DataCursor getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull DataStructure dsd, @Nonnull Key resource, boolean serieskeysonly) throws SdmxException, IOException; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index c548b274..6f4af4df 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -20,6 +20,7 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.util.Property.BoolProperty; +import it.bancaditalia.oss.sdmx.api.Codelist; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.Dataflow; @@ -64,6 +65,51 @@ public DataStructure toStructure(DataFlowStructure dsd) { return result.build(); } + public Dataflow fromFlowQuery(be.nbb.sdmx.facade.DataflowRef ref) { + Dataflow result = new Dataflow(); + result.setAgency(ref.getAgency()); + result.setId(ref.getId()); + result.setVersion(ref.getVersion()); + return result; + } + + public Dataflow fromFlow(be.nbb.sdmx.facade.Dataflow flow) { + Dataflow result = new Dataflow(); + result.setAgency(flow.getRef().getAgency()); + result.setId(flow.getRef().getId()); + result.setVersion(flow.getRef().getVersion()); + result.setDsdIdentifier(fromStructureRef(flow.getStructureRef())); + result.setName(flow.getLabel()); + return result; + } + + public DSDIdentifier fromStructureRef(be.nbb.sdmx.facade.DataStructureRef ref) { + return new DSDIdentifier(ref.getId(), ref.getAgency(), ref.getVersion()); + } + + public Dimension fromDimension(be.nbb.sdmx.facade.Dimension o) { + Dimension result = new Dimension(); + result.setId(o.getId()); + result.setPosition(o.getPosition()); + result.setName(o.getLabel()); + Codelist codelist = new Codelist(); + codelist.setCodes(o.getCodes()); + result.setCodeList(codelist); + return result; + } + + public DataFlowStructure fromStructure(DataStructure dsd) { + DataFlowStructure result = new DataFlowStructure(); + result.setAgency(dsd.getRef().getAgency()); + result.setId(dsd.getRef().getId()); + result.setVersion(dsd.getRef().getVersion()); + result.setName(dsd.getLabel()); + result.setTimeDimension(dsd.getTimeDimensionId()); + // FIXME: how to set MEASURE? + dsd.getDimensions().forEach(o -> result.setDimension(fromDimension(o))); + return result; + } + public it.bancaditalia.oss.sdmx.util.LanguagePriorityList fromLanguages(be.nbb.sdmx.facade.LanguagePriorityList l) { return it.bancaditalia.oss.sdmx.util.LanguagePriorityList.parse(l.toString()); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java index 319087dc..c58761a6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.ABS; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class AbsDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:abs:") - .supplier(ABS::new) - .entry("ABS", "Australian Bureau of Statistics", "sdmx:abs:http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") + .client(ConnectorRestClient.of(ABS::new)) + .entry("ABS", "Australian Bureau of Statistics", "http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java index ebe656aa..7267952a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.EUROSTAT; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class EurostatDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:eurostat:") - .supplier(EUROSTAT::new) - .entry("EUROSTAT", "Eurostat", "sdmx:eurostat:http://ec.europa.eu/eurostat/SDMX/diss-web/rest") + .client(ConnectorRestClient.of(EUROSTAT::new)) + .entry("EUROSTAT", "Eurostat", "http://ec.europa.eu/eurostat/SDMX/diss-web/rest") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java index cc325adc..5fa7c287 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.ILO; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class IloDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:ilo:") - .supplier(ILO::new) - .entry("ILO", "International Labour Office", "sdmx:ilo:https://www.ilo.org/ilostat/sdmx/ws/rest") + .client(ConnectorRestClient.of(ILO::new)) + .entry("ILO", "International Labour Office", "https://www.ilo.org/ilostat/sdmx/ws/rest") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java index dda79023..e918a9b4 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.IMF; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class ImfDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:imf:") - .supplier(IMF::new) - .entry("IMF", "International Monetary Fund", "sdmx:imf:http://sdmxws.imf.org/SDMXRest/sdmx.ashx") + .client(ConnectorRestClient.of(IMF::new)) + .entry("IMF", "International Monetary Fund", "http://sdmxws.imf.org/SDMXRest/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index e280f499..3485b5b7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -18,8 +18,8 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.util.HasCache; @@ -28,7 +28,6 @@ import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; @@ -36,17 +35,19 @@ import java.util.List; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorRestClient; import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; -import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; import internal.org.springframework.util.xml.XMLEventStreamReader; import internal.util.drivers.InseeDialect; +import internal.web.RestDriverSupport; import it.bancaditalia.oss.sdmx.api.Codelist; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.Dimension; import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; +import java.util.Map; import java.util.logging.Level; /** @@ -58,27 +59,22 @@ public final class InseeDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:insee:") - .supplier((u, i) -> new InseeClient(u)) - .entryPoint(INSEE_ENTRY) + .client(ConnectorRestClient.of(InseeClient::new)) + .entry("INSEE", "Institut national de la statistique et des études économiques", URL) .build(); @SdmxFix(id = "INSEE#1", cause = "Fallback to http due to some servers that use root certificate unknown to jdk'") - private static final SdmxWebEntryPoint INSEE_ENTRY = SdmxWebEntryPoint - .builder() - .name("INSEE") - .description("Institut national de la statistique et des études économiques") - .uri("sdmx:insee:http://bdm.insee.fr/series/sdmx") - .build(); + private static final String URL = "http://bdm.insee.fr/series/sdmx"; private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { @SdmxFix(id = "INSEE#2", cause = "Does not follow sdmx standard codes") private final SdmxDialect dialect; - private InseeClient(URI endpoint) { + private InseeClient(URI endpoint, Map properties) { super("", endpoint, false, false, true); this.dialect = new InseeDialect(); } @@ -92,9 +88,9 @@ public DataFlowStructure getDataFlowStructure(DSDIdentifier dsd, boolean full) t } @Override - public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { + public DataCursor getDataCursor(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return SeriesSupport.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SeriesSupport.asCursor(getData(flowRef, dsd, resource, serieskeysonly), resource); } @Override @@ -137,17 +133,16 @@ private void loadMissingCodes(Codelist codelist) throws SdmxException { } } - private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { + private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { return runQuery( getCompactData21Parser(dsd), - buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false), + buildDataQuery(Util.fromFlowQuery(flowRef), resource.toString(), null, null, serieskeysonly, null, false), SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); } - private Parser> getCompactData21Parser(DataFlowStructure dsd) { + private Parser> getCompactData21Parser(DataStructure dsd) { return (r, l) -> { - DataStructure tmp = Util.toStructure(dsd); - try (DataCursor cursor = SdmxXmlStreams.compactData21(tmp, dialect).parse(new XMLEventStreamReader(r), () -> { + try (DataCursor cursor = SdmxXmlStreams.compactData21(dsd, dialect).parse(new XMLEventStreamReader(r), () -> { })) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java index b8db52ee..db9a96f9 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.NBB; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class NbbDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:nbb:") - .supplier(NBB::new) - .entry("NBB", "National Bank Belgium", "sdmx:nbb:https://stat.nbb.be/restsdmx/sdmx.ashx") + .client(ConnectorRestClient.of(NBB::new)) + .entry("NBB", "National Bank Belgium", "https://stat.nbb.be/restsdmx/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java index 353f2ce7..f2e54ec5 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.OECD; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class OecdDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:oecd:") - .supplier(OECD::new) - .entry("OECD", "The Organisation for Economic Co-operation and Development", "sdmx:oecd:https://stats.oecd.org/restsdmx/sdmx.ashx") + .client(ConnectorRestClient.of(OECD::new)) + .entry("OECD", "The Organisation for Economic Co-operation and Development", "https://stats.oecd.org/restsdmx/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java index e58ed9f3..f9fc37fa 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java @@ -22,7 +22,8 @@ import java.util.Map; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; import java.net.URI; /** @@ -33,10 +34,10 @@ public final class Sdmx20Driver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:sdmx20:") - .supplier(Sdmx20Client::new) + .client(ConnectorRestClient.of(Sdmx20Client::new)) .build(); private static final class Sdmx20Client extends RestSdmx20Client { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index c23b53e2..2c7b43a4 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -17,6 +17,8 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.util.HasCache; @@ -24,8 +26,6 @@ import be.nbb.sdmx.facade.util.SdmxMediaType; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; @@ -36,11 +36,12 @@ import java.util.Map; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.ConnectorRestClient; import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; -import internal.connectors.ConnectorsDriverSupport; import internal.connectors.Util; import internal.org.springframework.util.xml.XMLEventStreamReader; +import internal.web.RestDriverSupport; import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; @@ -54,10 +55,10 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { private static final String PREFIX = "sdmx:sdmx21:"; @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix(PREFIX) - .supplier((u, i) -> new Sdmx21Client(u, Sdmx21Config.load(i))) + .client(ConnectorRestClient.of(Sdmx21Client::new)) .entryPoints(getEntryPoints()) .build(); @@ -182,15 +183,19 @@ private final static class Sdmx21Client extends RestSdmxClient implements HasDat private final Sdmx21Config config; + private Sdmx21Client(URI endpoint, Map properties) { + this(endpoint, Sdmx21Config.load(properties)); + } + private Sdmx21Client(URI endpoint, Sdmx21Config config) { super("", endpoint, config.isNeedsCredentials(), config.isNeedsURLEncoding(), config.isSupportsCompression()); this.config = config; } @Override - public DataCursor getDataCursor(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { + public DataCursor getDataCursor(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { // FIXME: avoid in-memory copy - return SeriesSupport.asCursor(getData(dataflow, dsd, resource, serieskeysonly), resource); + return SeriesSupport.asCursor(getData(flowRef, dsd, resource, serieskeysonly), resource); } @Override @@ -198,16 +203,16 @@ public boolean isSeriesKeysOnlySupported() { return config.isSeriesKeysOnlySupported(); } - private List getData(Dataflow dataflow, DataFlowStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { + private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { return runQuery( getCompactData21Parser(dsd), - buildDataQuery(dataflow, resource.toString(), null, null, serieskeysonly, null, false), + buildDataQuery(Util.fromFlowQuery(flowRef), resource.toString(), null, null, serieskeysonly, null, false), SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); } - private Parser> getCompactData21Parser(DataFlowStructure dsd) { + private Parser> getCompactData21Parser(DataStructure dsd) { return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(Util.toStructure(dsd)).parse(new XMLEventStreamReader(r), () -> { + try (DataCursor cursor = SdmxXmlStreams.compactData21(dsd).parse(new XMLEventStreamReader(r), () -> { })) { return SeriesSupport.copyOf(cursor); } catch (IOException ex) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java index dcf4a086..b05d3100 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java @@ -20,7 +20,8 @@ import it.bancaditalia.oss.sdmx.client.custom.UIS; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.connectors.ConnectorRestClient; +import internal.web.RestDriverSupport; /** * @@ -30,10 +31,10 @@ public final class UisDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate - private final ConnectorsDriverSupport support = ConnectorsDriverSupport + private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:uis:") - .supplier(UIS::new) - .entry("UIS", "Unesco Institute for Statistics", "sdmx:uis:http://data.uis.unesco.org/RestSDMX/sdmx.ashx") + .client(ConnectorRestClient.of(UIS::new)) + .entry("UIS", "Unesco Institute for Statistics", "http://data.uis.unesco.org/RestSDMX/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java new file mode 100644 index 00000000..76b0dc38 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java @@ -0,0 +1,162 @@ +/* + * Copyright 2015 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataQueryDetail; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.util.TtlCache; +import be.nbb.sdmx.facade.util.TypedId; +import java.io.IOException; +import java.time.Clock; +import java.util.List; +import java.util.concurrent.ConcurrentMap; + +/** + * + * @author Philippe Charles + */ +final class CachedRestClient implements RestClient { + + static CachedRestClient of(RestClient delegate, String base, ConcurrentMap cache, Clock clock, long ttlInMillis) { + return new CachedRestClient(delegate, base, cache, clock, ttlInMillis); + } + + private final RestClient delegate; + private final TtlCache cache; + private final TypedId> idOfFlows; + private final TypedId idOfFlow; + private final TypedId idOfStruct; + private final TypedId idOfKeysOnly; + + CachedRestClient(RestClient delegate, String base, ConcurrentMap cache, Clock clock, long ttlInMillis) { + this.delegate = delegate; + this.cache = TtlCache.of(cache, clock, ttlInMillis); + this.idOfFlows = TypedId.of("flows://" + base); + this.idOfFlow = TypedId.of("flow://" + base); + this.idOfStruct = TypedId.of("struct://" + base); + this.idOfKeysOnly = TypedId.of("keys://" + base); + } + + @Override + public List getFlows() throws IOException { + return loadDataFlowsWithCache(); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + Dataflow result = peekDataflowFromCache(ref); + return result != null ? result : loadDataflowWithCache(ref); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + return loadDataStructureWithCache(ref); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + if (!query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)) { + return delegate.getData(flowRef, dsd, query); + } + return loadKeysOnlyWithCache(flowRef, dsd, query).getCursor(flowRef, query) + .orElseThrow(() -> new IOException("Data not found")); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return delegate.isSeriesKeysOnlySupported(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) { + return delegate.peekStructureRef(flowRef); + } + + private List loadDataFlowsWithCache() throws IOException { + List result = cache.get(idOfFlows); + if (result == null) { + result = delegate.getFlows(); + cache.put(idOfFlows, result); + } + return result; + } + + private DataStructure loadDataStructureWithCache(DataStructureRef ref) throws IOException { + TypedId id = idOfStruct.with(ref); + DataStructure result = cache.get(id); + if (result == null) { + result = delegate.getStructure(ref); + cache.put(id, result); + } + return result; + } + + private SdmxRepository loadKeysOnlyWithCache(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + TypedId id = idOfKeysOnly.with(flowRef); + SdmxRepository result = cache.get(id); + if (result == null || isBroaderRequest(query.getKey(), result)) { + result = copyDataKeys(flowRef, dsd, query); + cache.put(id, result); + } + return result; + } + + private Dataflow peekDataflowFromCache(DataflowRef ref) { + // check if dataflow has been already loaded by #loadDataFlowsById + List dataFlows = cache.get(idOfFlows); + if (dataFlows == null) { + return null; + } + for (Dataflow o : dataFlows) { + // FIXME: use #contains instead of #id + if (o.getRef().getId().equals(ref.getId())) { + return o; + } + } + return null; + } + + private Dataflow loadDataflowWithCache(DataflowRef ref) throws IOException { + TypedId id = idOfFlow.with(ref); + Dataflow result = cache.get(id); + if (result == null) { + result = delegate.getFlow(ref); + cache.put(id, result); + } + return result; + } + + private boolean isBroaderRequest(Key key, SdmxRepository repo) { + return key.supersedes(Key.parse(repo.getName())); + } + + private SdmxRepository copyDataKeys(DataflowRef flowRef, DataStructure structure, DataQuery query) throws IOException { + try (DataCursor cursor = delegate.getData(flowRef, structure, query)) { + return SdmxRepository.builder() + .copyOf(flowRef, cursor) + .name(query.getKey().toString()) + .build(); + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java new file mode 100644 index 00000000..6de6a88c --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java @@ -0,0 +1,128 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.util.UnexpectedIOException; +import java.io.IOException; +import java.util.List; +import java.util.logging.Level; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor(staticName = "of") +@lombok.extern.java.Log +final class FailsafeRestClient implements RestClient { + + private final RestClient delegate; + + @Override + public List getFlows() throws IOException { + List result; + + try { + result = delegate.getFlows(); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting datasets"); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting datasets"); + } + + return result; + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + Dataflow result; + + try { + result = delegate.getFlow(ref); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting details from dataset '%s'", ref); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting details from dataset '%s'", ref); + } + + return result; + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + DataStructure result; + + try { + result = delegate.getStructure(ref); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting data structure from dataset '%s'", ref); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting data structure from dataset '%s'", ref); + } + + return result; + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + DataCursor result; + + try { + result = delegate.getData(flowRef, dsd, query); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting data from dataset '%s' with key '%s'", flowRef, query.getKey()); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting data from dataset '%s' with key '%s'", flowRef, query.getKey()); + } + + return result; + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return delegate.isSeriesKeysOnlySupported(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) { + return delegate.peekStructureRef(flowRef); + } + + private static IOException unexpected(RuntimeException ex, String format, Object... args) { + log.log(Level.WARNING, format, args); + return new UnexpectedIOException(ex); + } + + private static IOException unexpectedNull(String format, Object... args) { + String msg = String.format(format, args); + log.log(Level.WARNING, msg); + return new UnexpectedIOException(new NullPointerException(msg)); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java new file mode 100644 index 00000000..cd966882 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java @@ -0,0 +1,60 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import java.io.IOException; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * + * @author Philippe Charles + */ +public interface RestClient { + + @Nonnull + List getFlows() throws IOException; + + @Nonnull + Dataflow getFlow(@Nonnull DataflowRef ref) throws IOException; + + @Nonnull + DataStructure getStructure(@Nonnull DataStructureRef ref) throws IOException; + + @Nonnull + DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataStructure dsd, @Nonnull DataQuery query) throws IOException; + + boolean isSeriesKeysOnlySupported(); + + @Nullable + DataStructureRef peekStructureRef(@Nonnull DataflowRef flowRef); + + interface Supplier { + + @Nonnull + RestClient get(@Nonnull SdmxWebEntryPoint entryPoint, @Nonnull String prefix, @Nonnull LanguagePriorityList languages); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java similarity index 63% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java index dc57430e..90c3362e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.web; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -22,76 +22,65 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.DataQueryDetail; import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; -import java.util.Map; +import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nonnull; /** * * @author Philippe Charles */ -final class ConnectorsConnection implements SdmxConnection { +@lombok.RequiredArgsConstructor(staticName = "of") +final class RestConnection implements SdmxConnection { - interface Resource { - - @Nonnull - Map loadDataFlowsById() throws IOException; - - @Nonnull - it.bancaditalia.oss.sdmx.api.Dataflow loadDataflow(@Nonnull DataflowRef flowRef) throws IOException; - - @Nonnull - it.bancaditalia.oss.sdmx.api.DataFlowStructure loadDataStructure(@Nonnull DataflowRef flowRef) throws IOException; - - @Nonnull - DataCursor loadData(@Nonnull DataflowRef flowRef, @Nonnull Key key, boolean serieskeysonly) throws IOException; - - boolean isSeriesKeysOnlySupported(); - } - - private final Resource resource; - private boolean closed; - - ConnectorsConnection(Resource resource) { - this.resource = resource; - this.closed = false; - } + @lombok.NonNull + private final RestClient client; + private boolean closed = false; @Override public Set getFlows() throws IOException { checkState(); - return resource.loadDataFlowsById().values().stream() - .map(Util::toFlow) - .collect(Collectors.toSet()); + return new HashSet<>(client.getFlows()); } @Override public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); - return Util.toFlow(resource.loadDataflow(flowRef)); + return client.getFlow(flowRef); } @Override public DataStructure getStructure(DataflowRef flowRef) throws IOException { checkState(); - return Util.toStructure(resource.loadDataStructure(flowRef)); + + DataStructureRef structRef = client.peekStructureRef(flowRef); + if (structRef == null) { + Dataflow flow = client.getFlow(flowRef); + structRef = flow.getStructureRef(); + } + + return client.getStructure(structRef); } @Override public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); - boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); - if (serieskeysonly && !isSeriesKeysOnlySupported()) { - throw new IllegalStateException("serieskeysonly not supported"); + checkQuery(query); + + DataStructureRef structRef = client.peekStructureRef(flowRef); + if (structRef == null) { + Dataflow flow = client.getFlow(flowRef); + structRef = flow.getStructureRef(); + flowRef = flow.getRef(); // FIXME: all,...,latest fails sometimes } - return resource.loadData(flowRef, query.getKey(), serieskeysonly); + + DataStructure structure = client.getStructure(structRef); + return client.getData(flowRef, structure, query); } @Override @@ -101,7 +90,7 @@ public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOE @Override public boolean isSeriesKeysOnlySupported() { - return resource.isSeriesKeysOnlySupported(); + return client.isSeriesKeysOnlySupported(); } @Override @@ -114,4 +103,11 @@ private void checkState() throws IOException { throw new IOException("Connection closed"); } } + + private void checkQuery(DataQuery query) { + boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); + if (serieskeysonly && !isSeriesKeysOnlySupported()) { + throw new IllegalStateException("serieskeysonly not supported"); + } + } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestDriverSupport.java similarity index 52% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java rename to sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestDriverSupport.java index dce50b2d..3a338e37 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorsDriverSupport.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestDriverSupport.java @@ -14,28 +14,23 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.web; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CACHE_TTL; -import static be.nbb.sdmx.facade.util.CommonSdmxProperty.CONNECT_TIMEOUT; -import static be.nbb.sdmx.facade.util.CommonSdmxProperty.READ_TIMEOUT; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.time.Clock; import java.util.Collection; -import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; @@ -45,13 +40,13 @@ */ @lombok.Builder(builderClassName = "Builder") @ThreadSafe -public final class ConnectorsDriverSupport implements SdmxWebDriver, HasCache { +public final class RestDriverSupport implements SdmxWebDriver, HasCache { @lombok.NonNull private final String prefix; @lombok.NonNull - private final BiFunction, GenericSDMXClient> supplier; + private final RestClient.Supplier client; @lombok.Singular private final Collection entryPoints; @@ -64,7 +59,9 @@ public final class ConnectorsDriverSupport implements SdmxWebDriver, HasCache { @Override public SdmxConnection connect(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { - return new ConnectorsConnection(getResource(entryPoint, languages)); + Objects.requireNonNull(entryPoint); + Objects.requireNonNull(languages); + return RestConnection.of(getClient(entryPoint, languages)); } @Override @@ -87,28 +84,22 @@ public void setCache(ConcurrentMap cache) { cacheSupport.setCache(cache); } - private ConnectorsConnection.Resource getResource(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { - return getResource(getEndpoint(entryPoint, prefix), entryPoint.getProperties(), languages); + private RestClient getClient(SdmxWebEntryPoint entryPoint, LanguagePriorityList languages) throws IOException { + RestClient origin = client.get(entryPoint, prefix, languages); + RestClient cached = CachedRestClient.of(origin, getBase(entryPoint, prefix, languages), getCache(), clock, getCacheTtl(entryPoint)); + return FailsafeRestClient.of(cached); } - private ConnectorsConnection.Resource getResource(URI endpoint, Map info, LanguagePriorityList languages) throws IOException { - GenericSDMXClient client = supplier.apply(endpoint, info); - configure(client, info, languages); - return CachedResource.of(client, endpoint, languages, getCache(), clock, CACHE_TTL.get(info, DEFAULT_CACHE_TTL)); + private long getCacheTtl(SdmxWebEntryPoint entryPoint) { + return CACHE_TTL.get(entryPoint.getProperties(), DEFAULT_CACHE_TTL); } - private static void configure(GenericSDMXClient client, Map info, LanguagePriorityList languages) { - if (client instanceof RestSdmxClient) { - ((RestSdmxClient) client).setLanguages(Util.fromLanguages(languages)); - ((RestSdmxClient) client).setConnectTimeout(CONNECT_TIMEOUT.get(info, DEFAULT_CONNECT_TIMEOUT)); - ((RestSdmxClient) client).setReadTimeout(READ_TIMEOUT.get(info, DEFAULT_READ_TIMEOUT)); - } - } - - private final static int DEFAULT_CONNECT_TIMEOUT = 1000 * 60 * 2; // 2 minutes - private final static int DEFAULT_READ_TIMEOUT = 1000 * 60 * 2; // 2 minutes private static final long DEFAULT_CACHE_TTL = TimeUnit.MINUTES.toMillis(5); + private static String getBase(SdmxWebEntryPoint entryPoint, String prefix, LanguagePriorityList languages) throws IOException { + return getEndpoint(entryPoint, prefix).getHost() + languages.toString() + "/"; + } + @Nonnull public static URI getEndpoint(@Nonnull SdmxWebEntryPoint o, @Nonnull String prefix) throws IOException { try { @@ -118,37 +109,10 @@ public static URI getEndpoint(@Nonnull SdmxWebEntryPoint o, @Nonnull String pref } } - @FunctionalInterface - public interface ClientConstructor { - - @Nonnull - GenericSDMXClient get() throws URISyntaxException; - } - public static final class Builder { - public Builder supplier(@Nonnull ClientConstructor constructor) { - this.supplier = (x, y) -> newClient(constructor, x); - return this; - } - - public Builder supplier(@Nonnull BiFunction, GenericSDMXClient> supplier) { - this.supplier = supplier; - return this; - } - public Builder entry(@Nonnull String name, @Nonnull String description, @Nonnull String url) { - return entryPoint(SdmxWebEntryPoint.builder().name(name).description(description).uri(url).build()); - } - - private static GenericSDMXClient newClient(ClientConstructor constructor, URI endpoint) { - try { - GenericSDMXClient result = constructor.get(); - result.setEndpoint(endpoint); - return result; - } catch (URISyntaxException ex) { - throw new RuntimeException(ex); - } + return entryPoint(SdmxWebEntryPoint.builder().name(name).description(description).uri(prefix + url).build()); } } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java deleted file mode 100644 index c172bb89..00000000 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsDriverSupportTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import org.junit.Test; - -/** - * - * @author Philippe Charles - */ -public class ConnectorsDriverSupportTest { - - @Test - public void testFactories() { - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java index b53a766e..db5e176d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/UtilTest.java @@ -16,9 +16,15 @@ */ package internal.connectors; -import it.bancaditalia.oss.sdmx.exceptions.SdmxExceptionFactory; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Dimension; +import static internal.connectors.Util.*; +import static it.bancaditalia.oss.sdmx.exceptions.SdmxExceptionFactory.createRestException; import java.net.HttpURLConnection; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -29,7 +35,38 @@ public class UtilTest { @Test public void testIsNoResultMatchingQuery() { - assertThat(Util.isNoResultMatchingQuery(SdmxExceptionFactory.createRestException(HttpURLConnection.HTTP_NOT_FOUND, null, null))).isTrue(); - assertThat(Util.isNoResultMatchingQuery(SdmxExceptionFactory.createRestException(HttpURLConnection.HTTP_BAD_REQUEST, null, null))).isFalse(); + assertThat(isNoResultMatchingQuery(createRestException(HttpURLConnection.HTTP_NOT_FOUND, null, null))).isTrue(); + assertThat(isNoResultMatchingQuery(createRestException(HttpURLConnection.HTTP_BAD_REQUEST, null, null))).isFalse(); + } + + @Test + public void testFlow() { + Dataflow o = Dataflow.of(DataflowRef.parse("flow1"), DataStructureRef.parse("struct1"), "label0"); + assertThat(toFlow(fromFlow(o))).isEqualTo(o); + } + + @Test + public void testStructureRef() { + DataStructureRef o = DataStructureRef.parse("struct1"); + assertThat(toStructureRef(fromStructureRef(o))).isEqualTo(o); + } + + @Test + public void testDimension() { + Dimension o = Dimension.builder().id("1234").label("label1").position(1).code("hello", "world").build(); + assertThat(toDimension(fromDimension(o))).isEqualTo(o); + } + + @Test + public void testStructure() { + DataStructure o = DataStructure + .builder() + .ref(DataStructureRef.parse("struct1")) + .dimension(Dimension.builder().id("1234").label("label1").position(1).code("hello", "world").build()) + .label("label2") + .primaryMeasureId("OBS_VALUE") + .timeDimensionId("obs_period") + .build(); + assertThat(toStructure(fromStructure(o))).isEqualTo(o); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java similarity index 50% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java index 5fc9734d..42bc08d1 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/CachedResourceTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java @@ -14,126 +14,123 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.web; +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.NoOpCursor; import be.nbb.sdmx.facade.util.TypedId; -import it.bancaditalia.oss.sdmx.api.DSDIdentifier; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.api.Dataflow; -import it.bancaditalia.oss.sdmx.api.GenericSDMXClient; -import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import java.io.IOException; -import java.net.URI; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatNullPointerException; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** * * @author Philippe Charles */ -public class CachedResourceTest { +public class CachedRestClientTest { @Test - public void testLoadDataFlowsById() throws IOException { + public void testGetFlows() throws IOException { CountingClient resource = new CountingClient(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedResource target = new CachedResource(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); - target.loadDataFlowsById(); + target.getFlows(); assertThat(resource.count).isEqualTo(1); assertThat(cache).containsOnlyKeys(flowsId); - target.loadDataFlowsById(); + target.getFlows(); assertThat(resource.count).isEqualTo(1); assertThat(cache).containsOnlyKeys(flowsId); clock.plus(100); - target.loadDataFlowsById(); + target.getFlows(); assertThat(resource.count).isEqualTo(2); assertThat(cache).containsOnlyKeys(flowsId); cache.clear(); - target.loadDataFlowsById(); + target.getFlows(); assertThat(resource.count).isEqualTo(3); assertThat(cache).containsOnlyKeys(flowsId); } @Test - public void testLoadDataflow() throws IOException { + public void testGetFlow() throws IOException { CountingClient resource = new CountingClient(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedResource target = new CachedResource(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); - assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); + assertThatNullPointerException().isThrownBy(() -> target.getFlow(null)); - target.loadDataflow(sample); + target.getFlow(FLOW_REF); assertThat(resource.count).isEqualTo(1); assertThat(cache).containsOnlyKeys(flowId); - target.loadDataflow(sample); + target.getFlow(FLOW_REF); assertThat(resource.count).isEqualTo(1); assertThat(cache).containsOnlyKeys(flowId); clock.plus(100); - target.loadDataflow(sample); + target.getFlow(FLOW_REF); assertThat(resource.count).isEqualTo(2); assertThat(cache).containsOnlyKeys(flowId); cache.clear(); - target.loadDataflow(sample); + target.getFlow(FLOW_REF); assertThat(resource.count).isEqualTo(3); assertThat(cache).containsOnlyKeys(flowId); cache.clear(); - target.loadDataFlowsById(); - target.loadDataflow(sample); + target.getFlows(); + target.getFlow(FLOW_REF); assertThat(resource.count).isEqualTo(4); assertThat(cache).containsOnlyKeys(flowsId); } @Test - public void testLoadDataStructure() throws IOException { + public void testGetStructure() throws IOException { CountingClient resource = new CountingClient(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedResource target = new CachedResource(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); - assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); + assertThatNullPointerException().isThrownBy(() -> target.getStructure(null)); - target.loadDataStructure(sample); - assertThat(resource.count).isEqualTo(2); - assertThat(cache).containsOnlyKeys(flowId, structId); + target.getStructure(STRUCT_REF); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(structId); - target.loadDataStructure(sample); - assertThat(resource.count).isEqualTo(2); - assertThat(cache).containsOnlyKeys(flowId, structId); + target.getStructure(STRUCT_REF); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(structId); clock.plus(100); - target.loadDataStructure(sample); - assertThat(resource.count).isEqualTo(4); - assertThat(cache).containsOnlyKeys(flowId, structId); + target.getStructure(STRUCT_REF); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(structId); cache.clear(); - target.loadDataStructure(sample); - assertThat(resource.count).isEqualTo(6); - assertThat(cache).containsOnlyKeys(flowId, structId); + target.getStructure(STRUCT_REF); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(structId); } @Test @@ -142,30 +139,33 @@ public void testLoadData() throws IOException { ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedResource target = new CachedResource(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); - assertThatNullPointerException().isThrownBy(() -> target.loadDataflow(null)); + assertThatNullPointerException().isThrownBy(() -> target.getData(null, null, query)); - target.loadData(sample, Key.ALL, true); - assertThat(resource.count).isEqualTo(3); - assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + target.getData(FLOW_REF, null, query); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(keysId); - target.loadData(sample, Key.ALL, true); - assertThat(resource.count).isEqualTo(3); - assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + target.getData(FLOW_REF, null, query); + assertThat(resource.count).isEqualTo(1); + assertThat(cache).containsOnlyKeys(keysId); clock.plus(100); - target.loadData(sample, Key.ALL, true); - assertThat(resource.count).isEqualTo(6); - assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + target.getData(FLOW_REF, null, query); + assertThat(resource.count).isEqualTo(2); + assertThat(cache).containsOnlyKeys(keysId); cache.clear(); - target.loadData(sample, Key.ALL, true); - assertThat(resource.count).isEqualTo(9); - assertThat(cache).containsOnlyKeys(flowId, structId, keysId); + target.getData(FLOW_REF, null, query); + assertThat(resource.count).isEqualTo(3); + assertThat(cache).containsOnlyKeys(keysId); } - private final DataflowRef sample = DataflowRef.parse("sample"); + private static final DataflowRef FLOW_REF = DataflowRef.parse("sample"); + private static final DataStructureRef STRUCT_REF = DataStructureRef.parse("sample"); + private static final DataStructure STRUCT = DataStructure.builder().ref(STRUCT_REF).label("").build(); + private final DataQuery query = DataQuery.of(Key.ALL, true); private final TypedId flowsId = TypedId.of("flows://"); private final TypedId flowId = TypedId.of("flow://all,sample,latest"); private final TypedId structId = TypedId.of("struct://all,sample,latest"); @@ -195,62 +195,42 @@ public Instant instant() { } } - private static final class CountingClient implements GenericSDMXClient { + private static final class CountingClient implements RestClient { int count = 0; @Override - public Map getDataflows() throws SdmxException { + public List getFlows() throws IOException { count++; - return Collections.singletonMap("sample", new Dataflow()); + return Collections.singletonList(Dataflow.of(FLOW_REF, STRUCT_REF, "sample")); } @Override - public Dataflow getDataflow(String string, String string1, String string2) throws SdmxException { + public Dataflow getFlow(DataflowRef ref) throws IOException { count++; - return new Dataflow(); + return Dataflow.of(FLOW_REF, STRUCT_REF, "sample"); } @Override - public DataFlowStructure getDataFlowStructure(DSDIdentifier dsdi, boolean bln) throws SdmxException { + public DataStructure getStructure(DataStructureRef ref) throws IOException { count++; - return new DataFlowStructure(); + return STRUCT; } @Override - public Map getCodes(String string, String string1, String string2) throws SdmxException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public List getTimeSeries(Dataflow dtflw, DataFlowStructure dfs, String string, String string1, String string2, boolean bln, String string3, boolean bln1) throws SdmxException { + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { count++; - return Collections.emptyList(); - } - - @Override - public boolean needsCredentials() { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public void setCredentials(String string, String string1) { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public URI getEndpoint() throws SdmxException { - throw new UnsupportedOperationException("Not supported yet."); + return NoOpCursor.noOp(); } @Override - public void setEndpoint(URI url) { - throw new UnsupportedOperationException("Not supported yet."); + public boolean isSeriesKeysOnlySupported() { + return true; } @Override - public String buildDataURL(Dataflow dtflw, String string, String string1, String string2, boolean bln, String string3, boolean bln1) throws SdmxException { - throw new UnsupportedOperationException("Not supported yet."); + public DataStructureRef peekStructureRef(DataflowRef flowRef) { + return null; } } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java similarity index 83% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java index 879de953..2f6aff11 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsConnectionTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java @@ -14,21 +14,22 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package internal.connectors; +package internal.web; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.tck.ConnectionAssert; import org.junit.Test; +import test.NoOpRestClient; /** * * @author Philippe Charles */ -public class ConnectorsConnectionTest { +public class RestConnectionTest { @Test public void testCompliance() { DataflowRef ref = DataflowRef.parse("XYZ"); - ConnectionAssert.assertCompliance(() -> new ConnectorsConnection(null), ref); + ConnectionAssert.assertCompliance(() -> RestConnection.of(NoOpRestClient.INSTANCE), ref); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java index 18ad78fe..21b29faf 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java @@ -21,7 +21,7 @@ import be.nbb.sdmx.facade.util.IO; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.connectors.ConnectorsDriverSupport; +import internal.web.RestDriverSupport; import static org.assertj.core.api.Assertions.*; /** @@ -53,6 +53,6 @@ private void checkEntryPoint(SdmxWebEntryPoint o, String prefix) { assertThat(o.getDescription()).isNotBlank(); assertThat(o.getProperties()).isNotNull(); assertThat(o.getUri().toString()).startsWith(prefix); - assertThatCode(() -> ConnectorsDriverSupport.getEndpoint(o, prefix).toURL()).doesNotThrowAnyException(); + assertThatCode(() -> RestDriverSupport.getEndpoint(o, prefix).toURL()).doesNotThrowAnyException(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java new file mode 100644 index 00000000..a8eea5e3 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java @@ -0,0 +1,74 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import internal.web.RestClient; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * + * @author Philippe Charles + */ +public enum NoOpRestClient implements RestClient { + + INSTANCE; + + @Override + public List getFlows() throws IOException { + return Collections.emptyList(); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + Objects.requireNonNull(ref); + throw new IOException(); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + Objects.requireNonNull(ref); + throw new IOException(); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + Objects.requireNonNull(flowRef); + Objects.requireNonNull(dsd); + Objects.requireNonNull(query); + throw new IOException(); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return false; + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) { + Objects.requireNonNull(flowRef); + return null; + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java new file mode 100644 index 00000000..07ee8256 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import internal.web.RestClient; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * + * @author Philippe Charles + */ +@lombok.AllArgsConstructor(staticName = "of") +final class RepoRestClient implements RestClient { + + private final SdmxRepository repo; + + @Override + public List getFlows() throws IOException { + return new ArrayList(repo.getDataflows()); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + return repo.getFlow(ref).orElseThrow(IOException::new); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + return repo.getStructure(ref).orElseThrow(IOException::new); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + return repo.getCursor(flowRef, query).orElseThrow(IOException::new); + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return true; + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) { + return null; + } +} From e82810771bfba4064a6caee95039b31dc517612d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 27 Nov 2017 14:48:04 +0100 Subject: [PATCH 53/68] Improved code coverage. --- .../internal/file/SdmxFileConnectionImpl.java | 4 +- .../file/SdmxFileConnectionImplTest.java | 5 +- .../nbb/sdmx/facade/tck/ConnectionAssert.java | 53 ++++- .../nbb/sdmx/facade/repo/SdmxRepository.java | 2 +- .../main/java/be/nbb/sdmx/facade/util/IO.java | 194 ++++++++++++++--- .../sdmx/facade/repo/SdmxRepositoryTest.java | 79 ++++++- .../java/be/nbb/sdmx/facade/util/IOTest.java | 206 ++++++++++++++++++ .../connectors/ConnectorRestClient.java | 5 +- .../java/internal/web/CachedRestClient.java | 5 +- .../java/internal/web/FailsafeRestClient.java | 17 +- .../main/java/internal/web/RestClient.java | 4 +- .../java/internal/web/RestConnection.java | 4 +- .../internal/web/CachedRestClientTest.java | 139 +++++------- .../internal/web/FailsafeRestClientTest.java | 109 +++++++++ .../java/internal/web/RestConnectionTest.java | 12 +- .../src/test/java/test/FacadeResource.java | 74 ++++--- .../java/test/client/CallStackClient.java | 76 +++++++ .../java/test/client/FailingRestClient.java | 66 ++++++ .../test/{ => client}/NoOpRestClient.java | 15 +- .../test/java/test/client/NullRestClient.java | 73 +++++++ .../test/{ => client}/RepoRestClient.java | 18 +- 21 files changed, 959 insertions(+), 201 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java rename sdmx-facade/sdmx-facade-web/src/test/java/test/{ => client}/NoOpRestClient.java (90%) create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java rename sdmx-facade/sdmx-facade-web/src/test/java/test/{ => client}/RepoRestClient.java (77%) diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java index fb76bc13..076a3251 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java @@ -69,7 +69,7 @@ public Dataflow getFlow() throws IOException { } @Override - final public Dataflow getFlow(DataflowRef flowRef) throws IOException { + public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); checkFlowRef(flowRef); return dataflow; @@ -96,7 +96,7 @@ public DataCursor getCursor(DataQuery query) throws IOException { } @Override - final public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { + public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { checkState(); checkFlowRef(flowRef); Objects.requireNonNull(query); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index 96f42502..c6755d1e 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.IOException; import java.util.Optional; +import java.util.stream.Stream; import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -61,7 +62,9 @@ public void testFile() throws IOException { assertThat(conn.getStructure()).isEqualTo(conn.getStructure(files.asDataflowRef())); assertThatNullPointerException().isThrownBy(() -> conn.getCursor(null)); assertThatNullPointerException().isThrownBy(() -> conn.getStream(null)); - assertThat(conn.getStream(DataQuery.of(Key.ALL, false))).containsExactly(conn.getStream(DataQuery.of(Key.ALL, false)).toArray(Series[]::new)); + try (Stream stream = conn.getStream(DataQuery.of(Key.ALL, false))) { + assertThat(stream).containsExactly(conn.getStream(DataQuery.of(Key.ALL, false)).toArray(Series[]::new)); + } } @Test diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 53240f90..075a49ee 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -16,12 +16,17 @@ */ package be.nbb.sdmx.facade.tck; +import be.nbb.sdmx.facade.DataCursor; import internal.io.ConsumerWithIO; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.Series; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Callable; import org.assertj.core.api.SoftAssertions; @@ -44,6 +49,11 @@ public static void assertCompliance(Callable supplier, DataflowR public static void assertCompliance(SoftAssertions s, Callable supplier, DataflowRef ref) throws Exception { try (SdmxConnection conn = supplier.call()) { assertNonnull(s, conn, ref); + DataCursorAssert.assertCompliance(s, () -> conn.getCursor(ref, ALL)); + s.assertThat(conn.getStream(ref, ALL)).containsExactlyElementsOf(cursorToSeries(ref, ALL, conn)); + s.assertThat(conn.getFlows()).isNotEmpty().filteredOn(o -> ref.contains(o.getRef())).isNotEmpty(); + s.assertThat(conn.getFlow(ref)).isNotNull(); + s.assertThat(conn.getStructure(ref)).isNotNull(); } try (SdmxConnection conn = supplier.call()) { @@ -52,37 +62,37 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getCursor(ref, DataQuery.of(Key.ALL, false)), "getCursor(DataFlowRef, DataQuery)"); - assertState(s, supplier, o -> o.getStream(ref, DataQuery.of(Key.ALL, false)), "getStream(DataFlowRef, DataQuery)"); - assertState(s, supplier, o -> o.getStructure(ref), "getStructure(DataFlowRef)"); - assertState(s, supplier, o -> o.getFlow(ref), "getFlow(DataFlowRef)"); + assertState(s, supplier, o -> o.getCursor(ref, ALL), "getCursor(DataflowRef, DataQuery)"); + assertState(s, supplier, o -> o.getStream(ref, ALL), "getStream(DataflowRef, DataQuery)"); + assertState(s, supplier, o -> o.getStructure(ref), "getStructure(DataflowRef)"); + assertState(s, supplier, o -> o.getFlow(ref), "getFlow(DataflowRef)"); assertState(s, supplier, SdmxConnection::getFlows, "getFlows()"); } @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getCursor(null, DataQuery.of(Key.ALL, false))) - .as("Expecting 'getCursor(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getCursor(null, ALL)) + .as("Expecting 'getCursor(DataflowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getCursor(ref, null)) - .as("Expecting 'getCursor(DataFlowRef, DataQuery)' to raise NPE when called with null query") + .as("Expecting 'getCursor(DataflowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getStream(null, DataQuery.of(Key.ALL, false))) - .as("Expecting 'getStream(DataFlowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getStream(null, ALL)) + .as("Expecting 'getStream(DataflowRef, DataQuery)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getStream(ref, null)) - .as("Expecting 'getStream(DataFlowRef, DataQuery)' to raise NPE when called with null query") + .as("Expecting 'getStream(DataflowRef, DataQuery)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getStructure(null)) - .as("Expecting 'getStructure(DataFlowRef)' to raise NPE when called with null flowRef") + .as("Expecting 'getStructure(DataflowRef)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getFlow(null)) - .as("Expecting 'getFlow(DataFlowRef)' to raise NPE when called with null flowRef") + .as("Expecting 'getFlow(DataflowRef)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); } @@ -95,4 +105,23 @@ private static void assertState(SoftAssertions s, Callable suppl .hasMessageContaining("closed"); } } + + private static List cursorToSeries(DataflowRef ref, DataQuery query, SdmxConnection conn) throws IOException { + List result = new ArrayList(); + try (DataCursor c = conn.getCursor(ref, query)) { + while (c.nextSeries()) { + Series.Builder series = Series.builder(); + series.key(c.getSeriesKey()); + series.freq(c.getSeriesFrequency()); + series.meta(c.getSeriesAttributes()); + while (c.nextObs()) { + series.obs(Obs.of(c.getObsPeriod(), c.getObsValue())); + } + result.add(series.build()); + } + } + return result; + } + + private static final DataQuery ALL = DataQuery.of(Key.ALL, false); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 55ac526f..a61db293 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -81,7 +81,7 @@ public Optional getFlow(@Nonnull DataflowRef ref) { Objects.requireNonNull(ref); return dataflows .stream() - .filter(o -> o.getRef().equals(ref)) + .filter(o -> ref.contains(o.getRef())) .findFirst(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java index 3d55751a..dc541185 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java @@ -21,11 +21,13 @@ import java.io.UncheckedIOException; import java.util.Iterator; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * @@ -34,30 +36,166 @@ @lombok.experimental.UtilityClass public class IO { + @FunctionalInterface + public interface Runnable { + + void runWithIO() throws IOException; + + @Nonnull + default java.lang.Runnable asUnchecked() { + return () -> { + try { + runWithIO(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + @Nonnull + static Runnable throwing(@Nonnull java.util.function.Supplier ex) { + Objects.requireNonNull(ex); + return () -> { + throw ex.get(); + }; + } + + @Nonnull + static Runnable noOp() { + return () -> { + }; + } + } + @FunctionalInterface public interface Supplier { T getWithIO() throws IOException; + + @Nonnull + default java.util.function.Supplier asUnchecked() { + return () -> { + try { + return getWithIO(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + @Nonnull + static Supplier throwing(@Nonnull java.util.function.Supplier ex) { + Objects.requireNonNull(ex); + return () -> { + throw ex.get(); + }; + } + + @Nonnull + @SuppressWarnings("null") + static Supplier of(@Nullable T t) { + return () -> t; + } } @FunctionalInterface public interface Function { R applyWithIO(T t) throws IOException; + + @Nonnull + default java.util.function.Function asUnchecked() { + return (T t) -> { + try { + return applyWithIO(t); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + @Nonnull + static Function throwing(@Nonnull java.util.function.Supplier ex) { + Objects.requireNonNull(ex); + return o -> { + throw ex.get(); + }; + } + + @Nonnull + @SuppressWarnings("null") + static Function of(@Nullable R r) { + return o -> r; + } } @FunctionalInterface public interface Predicate { boolean testWithIO(T t) throws IOException; + + @Nonnull + default java.util.function.Predicate asUnchecked() { + return (T t) -> { + try { + return testWithIO(t); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + @Nonnull + static Predicate throwing(@Nonnull java.util.function.Supplier ex) { + Objects.requireNonNull(ex); + return o -> { + throw ex.get(); + }; + } + + @Nonnull + static Predicate of(boolean r) { + return o -> r; + } + } + + @FunctionalInterface + public interface Consumer { + + void acceptWithIO(T t) throws IOException; + + @Nonnull + default java.util.function.Consumer asUnchecked() { + return (T t) -> { + try { + acceptWithIO(t); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + @Nonnull + static Consumer throwing(@Nonnull java.util.function.Supplier ex) { + Objects.requireNonNull(ex); + return o -> { + throw ex.get(); + }; + } + + @Nonnull + static Consumer noOp() { + return o -> { + }; + } } @Nonnull - public Stream stream(IO.Supplier supplier, IO.Function> stream) throws IOException { + public Stream stream(@Nonnull Supplier supplier, @Nonnull Function> stream) throws IOException { T resource = supplier.getWithIO(); try { - return stream.applyWithIO(resource).onClose(asUncheckedRunnable(resource)); - } catch (Error | RuntimeException e) { + return stream.applyWithIO(resource).onClose(asUnchecked(resource::close)); + } catch (Error | RuntimeException | IOException e) { try { resource.close(); } catch (IOException ex) { @@ -71,8 +209,14 @@ public Stream stream(IO.Supplier supplier, IO.Fun } @Nonnull - public Stream streamNonnull(@Nonnull IO.Supplier nextSupplier) { - Iterator iter = new Iterator() { + public Stream streamNonnull(@Nonnull Supplier nextSupplier) { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(asIterator(nextSupplier), Spliterator.ORDERED | Spliterator.NONNULL), false); + } + + @Nonnull + private Iterator asIterator(@Nonnull Supplier nextSupplier) { + Objects.requireNonNull(nextSupplier); + return new Iterator() { T nextElement = null; @Override @@ -100,37 +244,25 @@ public T next() { } } }; - return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - iter, Spliterator.ORDERED | Spliterator.NONNULL), false); } - private Runnable asUncheckedRunnable(Closeable o) { - return () -> { - try { - o.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; + @Nonnull + public java.lang.Runnable asUnchecked(@Nonnull Runnable o) { + return o.asUnchecked(); } - public java.util.function.Function asUnchecked(Function o) { - return (T t) -> { - try { - return o.applyWithIO(t); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; + @Nonnull + public java.util.function.Function asUnchecked(@Nonnull Function o) { + return o.asUnchecked(); } - public java.util.function.Predicate asUnchecked(Predicate o) { - return (T t) -> { - try { - return o.testWithIO(t); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; + @Nonnull + public java.util.function.Predicate asUnchecked(@Nonnull Predicate o) { + return o.asUnchecked(); + } + + @Nonnull + public java.util.function.Consumer asUnchecked(@Nonnull Consumer o) { + return o.asUnchecked(); } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index 2774dd7a..c0c0dd43 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -16,6 +16,10 @@ */ package be.nbb.sdmx.facade.repo; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.DataflowRef; @@ -26,7 +30,7 @@ import be.nbb.sdmx.facade.util.SeriesSupport; import java.time.LocalDateTime; import java.util.Collections; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.*; import org.junit.Test; /** @@ -37,7 +41,7 @@ public class SdmxRepositoryTest { @Test public void testCompliance() { - ConnectionAssert.assertCompliance(repo::asConnection, xyz); + ConnectionAssert.assertCompliance(repo::asConnection, goodFlowRef); } @Test @@ -47,10 +51,75 @@ public void testDataCursorCompliance() { @Test public void testBuilder() { - assertThat(SdmxRepository.builder().name("test").data(xyz, series).build().isSeriesKeysOnlySupported()).isTrue(); + assertThat(SdmxRepository.builder().name("test").data(goodFlowRef, series).build().isSeriesKeysOnlySupported()).isTrue(); } - private final DataflowRef xyz = DataflowRef.parse("XYZ"); + @Test + @SuppressWarnings("null") + public void testGetCursor() { + assertThatNullPointerException().isThrownBy(() -> repo.getCursor(null, all)); + assertThatNullPointerException().isThrownBy(() -> repo.getCursor(goodFlowRef, null)); + + DataCursorAssert.assertCompliance(() -> repo.getCursor(goodFlowRef, all).get()); + + assertThat(repo.getCursor(goodFlowRef, all)).isNotEmpty(); + assertThat(repo.getCursor(badFlowRef, all)).isEmpty(); + } + + @Test + public void testGetData() { + assertThat(repo.getData()).hasSize(1).containsKey(goodFlowRef); + } + + @Test + @SuppressWarnings("null") + public void testGetFlow() { + assertThatNullPointerException().isThrownBy(() -> repo.getFlow(null)); + assertThat(repo.getFlow(goodFlowRef)).isNotEmpty(); + assertThat(repo.getFlow(badFlowRef)).isEmpty(); + assertThat(repo.getFlow(DataflowRef.of(null, "XYZ", null))).isNotEmpty(); + } + + @Test + public void testGetDataStructures() { + assertThat(repo.getDataStructures()).containsExactly(struct); + } + + @Test + public void testGetDataflows() { + assertThat(repo.getDataflows()).containsExactly(flow); + } + + @Test + @SuppressWarnings("null") + public void testGetStream() { + assertThatNullPointerException().isThrownBy(() -> repo.getStream(null, all)); + assertThatNullPointerException().isThrownBy(() -> repo.getStream(goodFlowRef, null)); + assertThat(repo.getStream(goodFlowRef, all)).isNotEmpty(); + assertThat(repo.getStream(badFlowRef, all)).isEmpty(); + } + + @Test + @SuppressWarnings("null") + public void testGetStructure() { + assertThatNullPointerException().isThrownBy(() -> repo.getStructure(null)); + assertThat(repo.getStructure(goodStructRef)).isNotEmpty(); + assertThat(repo.getStructure(badStructRef)).isEmpty(); + } + + private final DataStructureRef goodStructRef = DataStructureRef.of("NBB","goodStruct", "v1.0"); + private final DataStructureRef badStructRef = DataStructureRef.parse("badStruct"); + private final DataQuery all = DataQuery.of(Key.ALL, false); + private final DataflowRef goodFlowRef = DataflowRef.of("NBB", "XYZ", "v2.0"); + private final DataflowRef badFlowRef = DataflowRef.parse("other"); + private final Dataflow flow = Dataflow.of(goodFlowRef, goodStructRef, "flow1"); + private final DataStructure struct = DataStructure.builder().ref(goodStructRef).label("struct1").build(); private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); - private final SdmxRepository repo = SdmxRepository.builder().name("test").data(xyz, series).build(); + private final SdmxRepository repo = SdmxRepository + .builder() + .name("test") + .dataStructure(struct) + .dataflow(flow) + .data(goodFlowRef, series) + .build(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java new file mode 100644 index 00000000..27c9339c --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java @@ -0,0 +1,206 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class IOTest { + + @Test + @SuppressWarnings("null") + public void testStream() { + assertThatNullPointerException().isThrownBy(() -> IO.stream(null, emptyStream())); + assertThatNullPointerException().isThrownBy(() -> IO.stream(() -> null, null)); + + IO.Supplier factoryError = IO.Supplier.throwing(FactoryError::new); + IO.Supplier closeError = () -> IO.Runnable.throwing(CloseError::new)::runWithIO; + IO.Function> streamError = IO.Function.throwing(StreamError::new); + + assertThatThrownBy(() -> IO.stream(factoryError, emptyStream()).close()) + .isInstanceOf(FactoryError.class) + .hasNoSuppressedExceptions(); + + assertThatThrownBy(() -> IO.stream(factoryError, streamError).close()) + .isInstanceOf(FactoryError.class) + .hasNoSuppressedExceptions(); + + assertThatThrownBy(() -> IO.stream(closeError, emptyStream()).close()) + .isInstanceOf(UncheckedIOException.class) + .hasRootCauseInstanceOf(CloseError.class) + .hasNoSuppressedExceptions(); + + assertThatThrownBy(() -> IO.stream(closeError, streamError).close()) + .isInstanceOf(StreamError.class) + .hasSuppressedException(new CloseError()); + + assertThat(new AtomicBoolean(false)).satisfies(c -> { + assertThatThrownBy(() -> IO.stream(closeable(c), streamError).close()) + .isInstanceOf(StreamError.class) + .hasNoSuppressedExceptions(); + assertThat(c).isTrue(); + }); + + assertThat(new AtomicBoolean(false)).satisfies(c -> { + assertThatCode(() -> IO.stream(closeable(c), emptyStream()).close()).doesNotThrowAnyException(); + assertThat(c).isTrue(); + }); + + assertThat(new AtomicBoolean(false)).satisfies(c -> { + assertThatCode(() -> IO.stream(closeable(c), emptyStream())).doesNotThrowAnyException(); + assertThat(c).isFalse(); + }); + } + + @Test + @SuppressWarnings("null") + public void testStreamNonnull() { + assertThatNullPointerException().isThrownBy(() -> IO.streamNonnull(null)); + + assertThat(IO.streamNonnull(() -> null)).isEmpty(); + + Iterator iter = Arrays.asList("A", "B").iterator(); + assertThat(IO.streamNonnull(() -> iter.hasNext() ? iter.next() : null)).containsExactly("A", "B"); + + assertThatThrownBy(() -> IO.streamNonnull(IO.Supplier.throwing(FileNotFoundException::new)).count()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + @Test + @SuppressWarnings("null") + public void testShortcuts() { + assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Runnable) null)); + assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Function) null)); + assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Predicate) null)); + assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Consumer) null)); + } + + @Test + @SuppressWarnings("null") + public void testFunctionThrowing() { + assertThatNullPointerException().isThrownBy(() -> IO.Function.throwing(null)); + assertThatThrownBy(() -> IO.Function.throwing(FileNotFoundException::new).applyWithIO(null)) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testFunctionAsUnchecked() { + assertThatCode(() -> IO.Function.of("").asUnchecked().apply("")).doesNotThrowAnyException(); + + assertThatThrownBy(() -> IO.Function.throwing(FileNotFoundException::new).asUnchecked().apply(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + @Test + @SuppressWarnings("null") + public void testRunnableThrowing() { + assertThatNullPointerException().isThrownBy(() -> IO.Runnable.throwing(null)); + assertThatThrownBy(() -> IO.Runnable.throwing(FileNotFoundException::new).runWithIO()) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testRunnableAsUnchecked() { + assertThatCode(() -> IO.Runnable.noOp().asUnchecked().run()).doesNotThrowAnyException(); + + assertThatThrownBy(() -> IO.Runnable.throwing(FileNotFoundException::new).asUnchecked().run()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + @Test + @SuppressWarnings("null") + public void testSupplierThrowing() { + assertThatNullPointerException().isThrownBy(() -> IO.Supplier.throwing(null)); + assertThatThrownBy(() -> IO.Supplier.throwing(FileNotFoundException::new).getWithIO()) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testSupplierAsUnchecked() { + assertThatCode(() -> IO.Supplier.of("").asUnchecked().get()).doesNotThrowAnyException(); + + assertThatThrownBy(() -> IO.Supplier.throwing(FileNotFoundException::new).asUnchecked().get()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + @Test + @SuppressWarnings("null") + public void testPredicateThrowing() { + assertThatNullPointerException().isThrownBy(() -> IO.Predicate.throwing(null)); + assertThatThrownBy(() -> IO.Predicate.throwing(FileNotFoundException::new).testWithIO(null)) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testPredicateAsUnchecked() { + assertThatCode(() -> IO.Predicate.of(true).asUnchecked().test("")).doesNotThrowAnyException(); + + assertThatThrownBy(() -> IO.Predicate.throwing(FileNotFoundException::new).asUnchecked().test(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + @Test + @SuppressWarnings("null") + public void testConsumerThrowing() { + assertThatNullPointerException().isThrownBy(() -> IO.Consumer.throwing(null)); + assertThatThrownBy(() -> IO.Consumer.throwing(FileNotFoundException::new).acceptWithIO(null)) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testConsumerAsUnchecked() { + assertThatCode(() -> IO.Consumer.noOp().asUnchecked().accept("")).doesNotThrowAnyException(); + + assertThatThrownBy(() -> IO.Consumer.throwing(FileNotFoundException::new).asUnchecked().accept(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); + } + + private static IO.Function> emptyStream() { + return IO.Function.of(Stream.empty()); + } + + private static IO.Supplier closeable(AtomicBoolean o) { + return IO.Supplier.of(() -> o.set(true)); + } + + private static final class FactoryError extends IOException { + } + + private static final class CloseError extends IOException { + } + + private static final class StreamError extends IOException { + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java index 77a9ae49..59ecb2a0 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java @@ -82,6 +82,7 @@ public static RestClient.Supplier of(BiFunction, RestSdmxClient> }; } + @lombok.NonNull private final RestSdmxClient connector; @Override @@ -131,13 +132,13 @@ public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery quer } @Override - public boolean isSeriesKeysOnlySupported() { + public boolean isSeriesKeysOnlySupported() throws IOException { return connector instanceof HasSeriesKeysOnlySupported && ((HasSeriesKeysOnlySupported) connector).isSeriesKeysOnlySupported(); } @Override - public DataStructureRef peekStructureRef(DataflowRef ref) { + public DataStructureRef peekStructureRef(DataflowRef ref) throws IOException { return connector instanceof DotStat ? DataStructureRef.of(ref.getAgency(), ref.getId(), ref.getVersion()) : null; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java index 76b0dc38..1a579185 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java @@ -42,6 +42,7 @@ static CachedRestClient of(RestClient delegate, String base, ConcurrentMap cache return new CachedRestClient(delegate, base, cache, clock, ttlInMillis); } + @lombok.NonNull private final RestClient delegate; private final TtlCache cache; private final TypedId> idOfFlows; @@ -84,12 +85,12 @@ public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery quer } @Override - public boolean isSeriesKeysOnlySupported() { + public boolean isSeriesKeysOnlySupported() throws IOException { return delegate.isSeriesKeysOnlySupported(); } @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) { + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { return delegate.peekStructureRef(flowRef); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java index 6de6a88c..0a7709c8 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java @@ -35,6 +35,7 @@ @lombok.extern.java.Log final class FailsafeRestClient implements RestClient { + @lombok.NonNull private final RestClient delegate; @Override @@ -106,13 +107,21 @@ public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery quer } @Override - public boolean isSeriesKeysOnlySupported() { - return delegate.isSeriesKeysOnlySupported(); + public boolean isSeriesKeysOnlySupported() throws IOException { + try { + return delegate.isSeriesKeysOnlySupported(); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while checking keys-only support"); + } } @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) { - return delegate.peekStructureRef(flowRef); + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + try { + return delegate.peekStructureRef(flowRef); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while peeking struct ref for dataset '%s'", flowRef); + } } private static IOException unexpected(RuntimeException ex, String format, Object... args) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java index cd966882..a8f6a656 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java @@ -47,10 +47,10 @@ public interface RestClient { @Nonnull DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataStructure dsd, @Nonnull DataQuery query) throws IOException; - boolean isSeriesKeysOnlySupported(); + boolean isSeriesKeysOnlySupported() throws IOException; @Nullable - DataStructureRef peekStructureRef(@Nonnull DataflowRef flowRef); + DataStructureRef peekStructureRef(@Nonnull DataflowRef flowRef) throws IOException; interface Supplier { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java index 90c3362e..d2eac4b0 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java @@ -89,7 +89,7 @@ public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOE } @Override - public boolean isSeriesKeysOnlySupported() { + public boolean isSeriesKeysOnlySupported() throws IOException { return client.isSeriesKeysOnlySupported(); } @@ -104,7 +104,7 @@ private void checkState() throws IOException { } } - private void checkQuery(DataQuery query) { + private void checkQuery(DataQuery query) throws IOException { boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); if (serieskeysonly && !isSeriesKeysOnlySupported()) { throw new IllegalStateException("serieskeysonly not supported"); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java index 42bc08d1..932352eb 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java @@ -16,25 +16,23 @@ */ package internal.web; -import be.nbb.sdmx.facade.DataCursor; +import test.client.CallStackClient; import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.NoOpCursor; import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.*; import org.junit.Test; +import test.FacadeResource; +import static test.FacadeResource.ECB_FLOW_REF; +import static test.FacadeResource.ECB_STRUCT_REF; +import test.client.RepoRestClient; /** * @@ -44,132 +42,129 @@ public class CachedRestClientTest { @Test public void testGetFlows() throws IOException { - CountingClient resource = new CountingClient(); + AtomicInteger count = new AtomicInteger(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(getClient(count), "", cache, clock, 100); target.getFlows(); - assertThat(resource.count).isEqualTo(1); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(flowsId); target.getFlows(); - assertThat(resource.count).isEqualTo(1); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(flowsId); clock.plus(100); target.getFlows(); - assertThat(resource.count).isEqualTo(2); + assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(flowsId); cache.clear(); target.getFlows(); - assertThat(resource.count).isEqualTo(3); + assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(flowsId); } @Test public void testGetFlow() throws IOException { - CountingClient resource = new CountingClient(); + AtomicInteger count = new AtomicInteger(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(getClient(count), "", cache, clock, 100); assertThatNullPointerException().isThrownBy(() -> target.getFlow(null)); - target.getFlow(FLOW_REF); - assertThat(resource.count).isEqualTo(1); + target.getFlow(ECB_FLOW_REF); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(flowId); - target.getFlow(FLOW_REF); - assertThat(resource.count).isEqualTo(1); + target.getFlow(ECB_FLOW_REF); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(flowId); clock.plus(100); - target.getFlow(FLOW_REF); - assertThat(resource.count).isEqualTo(2); + target.getFlow(ECB_FLOW_REF); + assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(flowId); cache.clear(); - target.getFlow(FLOW_REF); - assertThat(resource.count).isEqualTo(3); + target.getFlow(ECB_FLOW_REF); + assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(flowId); cache.clear(); target.getFlows(); - target.getFlow(FLOW_REF); - assertThat(resource.count).isEqualTo(4); + target.getFlow(ECB_FLOW_REF); + assertThat(count).hasValue(4); assertThat(cache).containsOnlyKeys(flowsId); } @Test public void testGetStructure() throws IOException { - CountingClient resource = new CountingClient(); + AtomicInteger count = new AtomicInteger(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(getClient(count), "", cache, clock, 100); assertThatNullPointerException().isThrownBy(() -> target.getStructure(null)); - target.getStructure(STRUCT_REF); - assertThat(resource.count).isEqualTo(1); + target.getStructure(ECB_STRUCT_REF); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(structId); - target.getStructure(STRUCT_REF); - assertThat(resource.count).isEqualTo(1); + target.getStructure(ECB_STRUCT_REF); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(structId); clock.plus(100); - target.getStructure(STRUCT_REF); - assertThat(resource.count).isEqualTo(2); + target.getStructure(ECB_STRUCT_REF); + assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(structId); cache.clear(); - target.getStructure(STRUCT_REF); - assertThat(resource.count).isEqualTo(3); + target.getStructure(ECB_STRUCT_REF); + assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(structId); } @Test public void testLoadData() throws IOException { - CountingClient resource = new CountingClient(); + AtomicInteger count = new AtomicInteger(); ConcurrentHashMap cache = new ConcurrentHashMap(); FakeClock clock = new FakeClock(); - CachedRestClient target = new CachedRestClient(resource, "", cache, clock, 100); + CachedRestClient target = new CachedRestClient(getClient(count), "", cache, clock, 100); assertThatNullPointerException().isThrownBy(() -> target.getData(null, null, query)); - target.getData(FLOW_REF, null, query); - assertThat(resource.count).isEqualTo(1); + target.getData(ECB_FLOW_REF, null, query); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); - target.getData(FLOW_REF, null, query); - assertThat(resource.count).isEqualTo(1); + target.getData(ECB_FLOW_REF, null, query); + assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); clock.plus(100); - target.getData(FLOW_REF, null, query); - assertThat(resource.count).isEqualTo(2); + target.getData(ECB_FLOW_REF, null, query); + assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(keysId); cache.clear(); - target.getData(FLOW_REF, null, query); - assertThat(resource.count).isEqualTo(3); + target.getData(ECB_FLOW_REF, null, query); + assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(keysId); } - private static final DataflowRef FLOW_REF = DataflowRef.parse("sample"); - private static final DataStructureRef STRUCT_REF = DataStructureRef.parse("sample"); - private static final DataStructure STRUCT = DataStructure.builder().ref(STRUCT_REF).label("").build(); private final DataQuery query = DataQuery.of(Key.ALL, true); private final TypedId flowsId = TypedId.of("flows://"); - private final TypedId flowId = TypedId.of("flow://all,sample,latest"); - private final TypedId structId = TypedId.of("struct://all,sample,latest"); - private final TypedId keysId = TypedId.of("keys://all,sample,latest"); + private final TypedId flowId = TypedId.of("flow://").with(ECB_FLOW_REF); + private final TypedId structId = TypedId.of("struct://").with(ECB_STRUCT_REF); + private final TypedId keysId = TypedId.of("keys://").with(ECB_FLOW_REF); private static final class FakeClock extends Clock { @@ -195,42 +190,8 @@ public Instant instant() { } } - private static final class CountingClient implements RestClient { - - int count = 0; - - @Override - public List getFlows() throws IOException { - count++; - return Collections.singletonList(Dataflow.of(FLOW_REF, STRUCT_REF, "sample")); - } - - @Override - public Dataflow getFlow(DataflowRef ref) throws IOException { - count++; - return Dataflow.of(FLOW_REF, STRUCT_REF, "sample"); - } - - @Override - public DataStructure getStructure(DataStructureRef ref) throws IOException { - count++; - return STRUCT; - } - - @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { - count++; - return NoOpCursor.noOp(); - } - - @Override - public boolean isSeriesKeysOnlySupported() { - return true; - } - - @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) { - return null; - } + private static RestClient getClient(AtomicInteger count) throws IOException { + RestClient original = RepoRestClient.of(FacadeResource.ecb()); + return CallStackClient.of(original, count); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java new file mode 100644 index 00000000..2b866b72 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web; + +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.util.IO; +import be.nbb.sdmx.facade.util.UnexpectedIOException; +import java.io.IOException; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; +import test.FacadeResource; +import static test.FacadeResource.ECB_FLOW_REF; +import static test.FacadeResource.ECB_STRUCT_REF; +import test.client.FailingRestClient; +import test.client.NoOpRestClient; +import test.client.NullRestClient; +import test.client.RepoRestClient; + +/** + * + * @author Philippe Charles + */ +public class FailsafeRestClientTest { + + @Test + public void testGetFlows() { + assertOperation(o -> FailsafeRestClient.of(o).getFlows()); + } + + @Test + public void testGetFlow() { + assertOperation(o -> FailsafeRestClient.of(o).getFlow(ECB_FLOW_REF)); + } + + @Test + public void testGetStructure() { + assertOperation(o -> FailsafeRestClient.of(o).getStructure(ECB_STRUCT_REF)); + } + + @Test + public void testGetData() { + DataQuery all = DataQuery.of(Key.ALL, false); + DataStructure struct = DataStructure.builder().ref(ECB_STRUCT_REF).label("hello").build(); + + assertOperation(o -> FailsafeRestClient.of(o).getData(ECB_FLOW_REF, struct, all)); + } + + @Test + public void testIsSeriesKeysOnlySupported() { + IO.Consumer op = o -> FailsafeRestClient.of(o).isSeriesKeysOnlySupported(); + + assertThatThrownBy(() -> op.acceptWithIO(NoOpRestClient.INSTANCE)) + .isInstanceOf(IOException.class); + + assertThatThrownBy(() -> op.acceptWithIO(FailingRestClient.INSTANCE)) + .isInstanceOf(UnexpectedIOException.class) + .hasCauseInstanceOf(UnsupportedOperationException.class); + + assertThatCode(() -> op.acceptWithIO(RepoRestClient.of(FacadeResource.ecb()))) + .doesNotThrowAnyException(); + } + + @Test + public void testPeekStructureRef() { + IO.Consumer op = o -> FailsafeRestClient.of(o).peekStructureRef(ECB_FLOW_REF); + + assertThatThrownBy(() -> op.acceptWithIO(NoOpRestClient.INSTANCE)) + .isInstanceOf(IOException.class); + + assertThatThrownBy(() -> op.acceptWithIO(FailingRestClient.INSTANCE)) + .isInstanceOf(UnexpectedIOException.class) + .hasCauseInstanceOf(UnsupportedOperationException.class); + + assertThatCode(() -> op.acceptWithIO(RepoRestClient.of(FacadeResource.ecb()))) + .doesNotThrowAnyException(); + } + + private static void assertOperation(IO.Consumer op) { + assertThatThrownBy(() -> op.acceptWithIO(NoOpRestClient.INSTANCE)) + .isInstanceOf(IOException.class); + + assertThatThrownBy(() -> op.acceptWithIO(FailingRestClient.INSTANCE)) + .isInstanceOf(UnexpectedIOException.class) + .hasCauseInstanceOf(UnsupportedOperationException.class); + + assertThatThrownBy(() -> op.acceptWithIO(NullRestClient.INSTANCE)) + .isInstanceOf(UnexpectedIOException.class) + .hasCauseInstanceOf(NullPointerException.class); + + assertThatCode(() -> op.acceptWithIO(RepoRestClient.of(FacadeResource.ecb()))) + .doesNotThrowAnyException(); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java index 2f6aff11..5a114831 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java @@ -16,10 +16,12 @@ */ package internal.web; -import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.tck.ConnectionAssert; +import java.io.IOException; import org.junit.Test; -import test.NoOpRestClient; +import test.FacadeResource; +import test.client.RepoRestClient; /** * @@ -28,8 +30,8 @@ public class RestConnectionTest { @Test - public void testCompliance() { - DataflowRef ref = DataflowRef.parse("XYZ"); - ConnectionAssert.assertCompliance(() -> RestConnection.of(NoOpRestClient.INSTANCE), ref); + public void testCompliance() throws IOException { + SdmxRepository repo = FacadeResource.ecb(); + ConnectionAssert.assertCompliance(() -> RestConnection.of(RepoRestClient.of(repo)), FacadeResource.ECB_FLOW_REF); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java index 7da69bc9..404e118e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java @@ -18,6 +18,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; @@ -31,6 +32,7 @@ import internal.connectors.Util; import java.io.IOException; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import javax.xml.stream.XMLInputFactory; @@ -41,42 +43,58 @@ @lombok.experimental.UtilityClass public class FacadeResource { + public static final DataflowRef ECB_FLOW_REF = DataflowRef.of("ECB", "AME", "1.0"); + public static final DataStructureRef ECB_STRUCT_REF = DataStructureRef.of("ECB", "ECB_AME1", "1.0"); + public SdmxRepository nbb() throws IOException { - LanguagePriorityList l = LanguagePriorityList.parse("fr"); + SdmxRepository result = NBB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); - List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); + DataflowRef ref = DataflowRef.of("NBB", "TEST_DATASET", null); - DataflowRef ref = DataflowRef.of("NBB", "TEST_DATASET", null); + result = SdmxRepository.builder() + .dataStructures(structs) + .dataflows(flows) + .data(ref, data) + .name("NBB") + .seriesKeysOnlySupported(false) + .build(); - return SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) - .data(ref, data) - .name("NBB") - .seriesKeysOnlySupported(false) - .build(); + NBB.set(result); + } + return result; } public SdmxRepository ecb() throws IOException { - LanguagePriorityList l = LanguagePriorityList.parse("fr"); - - List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); - List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); - List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); - - DataflowRef ref = DataflowRef.of("ECB", "AME", "1.0"); - - return SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) - .data(ref, data) - .name("ECB") - .seriesKeysOnlySupported(true) - .build(); + SdmxRepository result = ECB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); + List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); + + result = SdmxRepository.builder() + .dataStructures(structs) + .dataflows(flows) + .data(ECB_FLOW_REF, data) + .name("ECB") + .seriesKeysOnlySupported(true) + .build(); + + ECB.set(result); + } + return result; } + private static final AtomicReference NBB = new AtomicReference<>(); + private static final AtomicReference ECB = new AtomicReference<>(); + private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { return SdmxXmlStreams.struct20(l).parseReader(f, xml::openReader); } @@ -115,6 +133,6 @@ private Dataflow asDataflow(DataStructure o) { DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); return Dataflow.of(ref, o.getRef(), o.getLabel()); } - + private final XMLInputFactory XIF = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java new file mode 100644 index 00000000..baf34c87 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test.client; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import internal.web.RestClient; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * + * @author Philippe Charles + */ +@lombok.RequiredArgsConstructor(staticName = "of") +public final class CallStackClient implements RestClient { + + @lombok.NonNull + private final RestClient delegate; + + @lombok.NonNull + private final AtomicInteger count; + + @Override + public List getFlows() throws IOException { + count.incrementAndGet(); + return delegate.getFlows(); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + count.incrementAndGet(); + return delegate.getFlow(ref); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + count.incrementAndGet(); + return delegate.getStructure(ref); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + count.incrementAndGet(); + return delegate.getData(flowRef, dsd, query); + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + return delegate.isSeriesKeysOnlySupported(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + return delegate.peekStructureRef(flowRef); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java new file mode 100644 index 00000000..53da214d --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test.client; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import internal.web.RestClient; +import java.io.IOException; +import java.util.List; + +/** + * + * @author Philippe Charles + */ +public enum FailingRestClient implements RestClient { + + INSTANCE; + + @Override + public List getFlows() throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java similarity index 90% rename from sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java index a8eea5e3..a2ea9165 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/NoOpRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package test; +package test.client; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataQuery; @@ -24,7 +24,6 @@ import be.nbb.sdmx.facade.DataflowRef; import internal.web.RestClient; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -35,10 +34,10 @@ public enum NoOpRestClient implements RestClient { INSTANCE; - + @Override public List getFlows() throws IOException { - return Collections.emptyList(); + throw new IOException(); } @Override @@ -62,13 +61,13 @@ public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery quer } @Override - public boolean isSeriesKeysOnlySupported() { - return false; + public boolean isSeriesKeysOnlySupported() throws IOException { + throw new IOException(); } @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) { + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { Objects.requireNonNull(flowRef); - return null; + throw new IOException(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java new file mode 100644 index 00000000..8b745f8b --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java @@ -0,0 +1,73 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test.client; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import internal.web.RestClient; +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * + * @author Philippe Charles + */ +public enum NullRestClient implements RestClient { + + INSTANCE; + + @Override + public List getFlows() throws IOException { + return null; + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + Objects.requireNonNull(ref); + return null; + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + Objects.requireNonNull(ref); + return null; + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + Objects.requireNonNull(flowRef); + Objects.requireNonNull(dsd); + Objects.requireNonNull(query); + return null; + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + return false; + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + Objects.requireNonNull(flowRef); + return null; + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java similarity index 77% rename from sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java index 07ee8256..0664fc20 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/RepoRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package test; +package test.client; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataQuery; @@ -33,8 +33,9 @@ * @author Philippe Charles */ @lombok.AllArgsConstructor(staticName = "of") -final class RepoRestClient implements RestClient { +public final class RepoRestClient implements RestClient { + @lombok.NonNull private final SdmxRepository repo; @Override @@ -44,26 +45,29 @@ public List getFlows() throws IOException { @Override public Dataflow getFlow(DataflowRef ref) throws IOException { - return repo.getFlow(ref).orElseThrow(IOException::new); + return repo.getFlow(ref) + .orElseThrow(() -> new IOException("Dataflow not found")); } @Override public DataStructure getStructure(DataStructureRef ref) throws IOException { - return repo.getStructure(ref).orElseThrow(IOException::new); + return repo.getStructure(ref) + .orElseThrow(() -> new IOException("DataStructure not found")); } @Override public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { - return repo.getCursor(flowRef, query).orElseThrow(IOException::new); + return repo.getCursor(flowRef, query) + .orElseThrow(() -> new IOException("Data not found")); } @Override - public boolean isSeriesKeysOnlySupported() { + public boolean isSeriesKeysOnlySupported() throws IOException { return true; } @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) { + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { return null; } } From 0a62450a198665122aa926399245365f28fc5e1b Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 29 Nov 2017 11:13:18 +0100 Subject: [PATCH 54/68] Moved IO & Stax to their own package. --- .../demetra/sdmx/file/SdmxFileProvider.java | 2 +- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 2 +- .../java/internal/sdmx/SdmxCubeAccessor.java | 2 +- .../java/internal/sdmx/SdmxCubeItems.java | 2 +- .../internal/connectors/TestResource.java | 2 +- .../nbb/sdmx/facade/file/SdmxFileManager.java | 2 +- .../internal/file/SdmxDecoderResource.java | 2 +- .../main/java/internal/file/SdmxFileUtil.java | 2 +- .../file/xml/DataStructureDecoder.java | 2 +- .../java/internal/file/xml/DataTypeProbe.java | 2 +- .../internal/file/xml/StaxSdmxDecoder.java | 2 +- .../file/SdmxFileConnectionImplTest.java | 2 +- .../file/xml/DataStructureDecoderTest.java | 2 +- .../internal/file/xml/DataTypeProbeTest.java | 2 +- .../nbb/sdmx/facade/util/SeriesSupport.java | 1 + .../facade/xml/stream/SdmxXmlStreams.java | 1 + .../stream/XMLStreamCompactDataCursor.java | 1 + .../stream/XMLStreamGenericDataCursor.java | 1 + .../xml/stream/XMLStreamStructure20.java | 1 + .../xml/stream/XMLStreamStructure21.java | 1 + .../be/nbb/{sdmx/facade => }/util/IO.java | 2 +- .../facade/xml/stream => util}/Stax.java | 5 +- .../sdmx/facade/util/SeriesSupportTest.java | 2 +- .../XMLStreamCompactDataCursorTest.java | 1 + .../XMLStreamGenericDataCursorTest.java | 1 + .../xml/stream/XMLStreamStructure20Test.java | 1 + .../xml/stream/XMLStreamStructure21Test.java | 1 + .../facade/xml/stream/XMLStreamUtilTest.java | 50 +++++++++++++++++++ .../be/nbb/{sdmx/facade => }/util/IOTest.java | 2 +- .../facade/xml/stream => util}/StaxTest.java | 24 +-------- .../internal/web/FailsafeRestClientTest.java | 2 +- .../test/java/test/ConnectorsResource.java | 2 +- .../src/test/java/test/DriverAssertions.java | 2 +- .../src/test/java/test/FacadeResource.java | 2 +- 34 files changed, 84 insertions(+), 47 deletions(-) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/{sdmx/facade => }/util/IO.java (99%) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/{sdmx/facade/xml/stream => util}/Stax.java (98%) create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/{sdmx/facade => }/util/IOTest.java (99%) rename sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/{sdmx/facade/xml/stream => util}/StaxTest.java (76%) diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 1b50f23f..83df214b 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -24,7 +24,7 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.file.SdmxFileManager; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import com.google.common.cache.Cache; import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index 4e600f54..1e9626c0 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -22,7 +22,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.cache.Cache; import ec.tss.ITsProvider; diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index 8c8a4d67..d3679576 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -21,7 +21,7 @@ import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import com.google.common.base.Converter; import com.google.common.collect.Maps; diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index fb550850..0d8c3d37 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -22,7 +22,7 @@ import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.HasFilePaths; diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java index 7cd228b3..b78c228f 100644 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java @@ -25,7 +25,7 @@ import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.samples.ByteSource; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.client.Parser; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 8a4a2d9b..10ccf06d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.file.CachedResource; import internal.file.SdmxDecoder; import internal.file.SdmxFileConnectionImpl; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java index 0a884c28..fbfcfc07 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java @@ -24,7 +24,7 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Optional; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 6564071a..4a7ce44b 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -17,7 +17,7 @@ package internal.file; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import java.io.File; import java.io.StringReader; import java.io.StringWriter; diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java index f30e27a2..99f08989 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java @@ -23,7 +23,7 @@ import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; /** * diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java index e742ecf5..f114a075 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java @@ -22,7 +22,7 @@ import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.file.SdmxDecoder; /** diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java index 1a174f31..506ce390 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java @@ -29,7 +29,7 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.util.List; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.file.SdmxDecoder; import internal.file.SdmxFileUtil; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index c6755d1e..ba4e03c5 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -29,7 +29,7 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import java.io.File; import java.io.IOException; import java.util.Optional; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index 60ffb49f..3ba7afb3 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -21,7 +21,7 @@ import java.io.IOException; import org.junit.Test; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import static org.assertj.core.api.Assertions.assertThat; import static internal.file.xml.CustomDataStructureBuilder.dimension; import javax.xml.stream.XMLInputFactory; diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java index 023d15ce..b5f96721 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java @@ -17,7 +17,7 @@ package internal.file.xml; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.file.SdmxDecoder; import java.io.IOException; import javax.xml.stream.XMLInputFactory; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index d3d463bd..9cda698c 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.util; +import be.nbb.util.IO; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index f02c1090..c8125734 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index 6ecb0c43..d646a30d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 8ae60459..5b842147 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index 208a938d..673554c6 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 5ca4d491..9da3dca3 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java similarity index 99% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java index dc541185..665deb0a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.util; import java.io.Closeable; import java.io.IOException; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java similarity index 98% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java index b1b7b2e4..1d6813dc 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java @@ -14,9 +14,8 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.xml.stream; +package be.nbb.util; -import be.nbb.sdmx.facade.util.IO; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; @@ -123,7 +122,7 @@ public XMLStreamIOException(String message, XMLStreamException ex) { } } - void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { + public void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { try { reader.close(); } catch (XMLStreamException ex) { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 171542b8..bb15f467 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -26,7 +26,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import java.io.IOException; import java.time.LocalDateTime; import java.util.Collections; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index dec2ec90..cc075913 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index ed89dc60..37383f8b 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index 4d373208..eea0a815 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index fdc95205..dca06284 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; +import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java new file mode 100644 index 00000000..d454d9d6 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class XMLStreamUtilTest { + + @Test + public void testIsTagMatch() { + assertThat(XMLStreamUtil.isTagMatch("", "")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "HelloWorld")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "helloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("helloWorld", "HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("", "HelloWorld")).isFalse(); + + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "HelloWorld")).isTrue(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "helloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "Hello")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "World")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "")).isFalse(); + + assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("helloWorld", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("Hello", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("World", "xxx:HelloWorld")).isFalse(); + assertThat(XMLStreamUtil.isTagMatch("", "xxx:HelloWorld")).isFalse(); + } + +} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java similarity index 99% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java index 27c9339c..5b0727ca 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/IOTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.util; +package be.nbb.util; import java.io.Closeable; import java.io.FileNotFoundException; diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java similarity index 76% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java rename to sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java index 6f8c26df..0427d526 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/StaxTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade.xml.stream; +package be.nbb.util; import java.io.Closeable; import java.io.IOException; @@ -31,28 +31,6 @@ */ public class StaxTest { - @Test - public void testIsTagMatch() { - assertThat(XMLStreamUtil.isTagMatch("", "")).isTrue(); - assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "HelloWorld")).isTrue(); - assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "helloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("helloWorld", "HelloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("", "HelloWorld")).isFalse(); - - assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "HelloWorld")).isTrue(); - assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "helloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "Hello")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "World")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("xxx:HelloWorld", "")).isFalse(); - - assertThat(XMLStreamUtil.isTagMatch("HelloWorld", "xxx:HelloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("helloWorld", "xxx:HelloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("Hello", "xxx:HelloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("World", "xxx:HelloWorld")).isFalse(); - assertThat(XMLStreamUtil.isTagMatch("", "xxx:HelloWorld")).isFalse(); - } - @Test @SuppressWarnings("null") public void testValidParseStream() throws IOException { diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java index 2b866b72..ad105329 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import java.io.IOException; import static org.assertj.core.api.Assertions.*; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java index 6854ede5..50e2818c 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java @@ -24,7 +24,7 @@ import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.ObsParser; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.connectors.PortableTimeSeriesCursor; import internal.connectors.Util; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java index 21b29faf..ff7c58b1 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java @@ -18,7 +18,7 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.util.IO; +import be.nbb.util.IO; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.web.RestDriverSupport; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java index 404e118e..df8a15e8 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java @@ -28,7 +28,7 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.sdmx.facade.xml.stream.Stax; +import be.nbb.util.Stax; import internal.connectors.Util; import java.io.IOException; import java.util.List; From 53cc5cd59d960b8025b3d7bff587e5aea86f75b7 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 29 Nov 2017 11:33:54 +0100 Subject: [PATCH 55/68] Fixed XXE vulnerability. --- .../src/main/java/be/nbb/util/Stax.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java index 1d6813dc..571fa87e 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java @@ -154,6 +154,16 @@ public boolean isNotNamespaceAware(@Nonnull XMLStreamReader f) { return !(Boolean) f.getProperty(XMLInputFactory.IS_NAMESPACE_AWARE); } + // https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Prevention_Cheat_Sheet#XMLInputFactory_.28a_StAX_parser.29 + public static void preventXXE(@Nonnull XMLInputFactory factory) { + if (factory.isPropertySupported(XMLInputFactory.SUPPORT_DTD)) { + factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + } + if (factory.isPropertySupported(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES)) { + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + } + } + private static final class ImmutableInputFactory extends XMLInputFactory { static final XMLInputFactory DEFAULT = new ImmutableInputFactory(true); @@ -166,6 +176,7 @@ private ImmutableInputFactory(boolean namespaceAware) { if (!namespaceAware && delegate.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE)) { delegate.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); } + preventXXE(delegate); } @Override From 268ba6ec7487b5a0e4f7d7bda06e75f617762cb4 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 29 Nov 2017 14:17:41 +0100 Subject: [PATCH 56/68] Added parsing of dataflow21. --- .../demetra/dotstat/DotStatAccessorTest.java | 28 ++-- .../internal/connectors/TestResource.java | 132 ---------------- .../java/internal/sdmx/SdmxQueryUtilTest.java | 68 ++++----- .../java/test/samples}/FacadeResource.java | 16 +- .../facade/xml/stream/SdmxXmlStreams.java | 6 + .../facade/xml/stream/XMLStreamFlow21.java | 144 ++++++++++++++++++ .../xml/stream/XMLStreamFlow21Test.java | 50 ++++++ .../PortableTimeSeriesCursorTest.java | 2 +- .../internal/web/CachedRestClientTest.java | 6 +- .../internal/web/FailsafeRestClientTest.java | 6 +- .../java/internal/web/RestConnectionTest.java | 2 +- .../{ => samples}/ConnectorsResource.java | 24 +-- .../java/test/samples/FacadeResource.java | 134 ++++++++++++++++ .../java/test/{ => samples}/ParsersTest.java | 2 +- 14 files changed, 411 insertions(+), 209 deletions(-) delete mode 100644 demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java rename {sdmx-facade/sdmx-facade-web/src/test/java/test => demetra-dotstat-core/src/test/java/test/samples}/FacadeResource.java (92%) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java create mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java rename sdmx-facade/sdmx-facade-web/src/test/java/test/{ => samples}/ConnectorsResource.java (87%) create mode 100644 sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java rename sdmx-facade/sdmx-facade-web/src/test/java/test/{ => samples}/ParsersTest.java (97%) diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index 248af315..eab6c659 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -19,13 +19,11 @@ import static be.nbb.demetra.dotstat.DotStatAccessor.getKey; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dimension; -import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.Series; -import internal.connectors.TestResource; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import com.google.common.base.Joiner; import ec.tss.tsproviders.db.DbAccessor; @@ -34,10 +32,15 @@ import ec.tstoolkit.timeseries.simplets.TsData; import ec.tstoolkit.timeseries.simplets.TsFrequency; import ec.tstoolkit.timeseries.simplets.TsPeriod; +import java.io.IOException; import java.util.Map; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; +import org.junit.BeforeClass; import org.junit.Test; +import test.samples.FacadeResource; +import static test.samples.FacadeResource.ECB_FLOW_REF; +import static test.samples.FacadeResource.NBB_FLOW_REF; /** * @@ -45,15 +48,20 @@ */ public class DotStatAccessorTest { - private final SdmxConnectionSupplier supplier = SdmxRepositoryManager.builder() - .repository(TestResource.nbb()) - .repository(TestResource.ecb()) - .build(); + private static SdmxConnectionSupplier supplier; + + @BeforeClass + public static void beforeClass() throws IOException { + supplier = SdmxRepositoryManager.builder() + .repository(FacadeResource.nbb()) + .repository(FacadeResource.ecb()) + .build(); + } private static DotStatBean nbbBean() { DotStatBean result = new DotStatBean(); result.setDbName("NBB"); - result.setFlowRef(DataflowRef.of("NBB", "TEST_DATASET", null)); + result.setFlowRef(NBB_FLOW_REF); result.setDimColumns(Joiner.on(',').join(new String[]{"SUBJECT", "LOCATION", "FREQUENCY"})); return result; } @@ -68,7 +76,7 @@ private static DotStatBean ecbBean() { DotStatBean result = new DotStatBean(); result.setDbName("ECB"); - result.setFlowRef(DataflowRef.parse("ECB,AME,1.0")); + result.setFlowRef(ECB_FLOW_REF); result.setDimColumns(Joiner.on(',').join(dimensions)); return result; } @@ -80,7 +88,7 @@ private static DbSetId ecbRoot() { @Test public void testGetKey() throws Exception { - DataStructure dfs = supplier.getConnection("NBB").getStructure(DataflowRef.of("NBB", "TEST_DATASET", null)); + DataStructure dfs = supplier.getConnection("NBB").getStructure(NBB_FLOW_REF); Map dimById = DotStatAccessor.dimensionById(dfs); // default ordering of dimensions @@ -101,7 +109,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { assertThat(supplier.getConnection("NBB") - .getStream(DataflowRef.of("NBB", "TEST_DATASET", null), DataQuery.of(Key.ALL, true)) + .getStream(NBB_FLOW_REF, DataQuery.of(Key.ALL, true)) .map(Series::getKey)).contains(Key.parse("LOCSTL04.AUS.M")); } diff --git a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java b/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java deleted file mode 100644 index b78c228f..00000000 --- a/demetra-dotstat-core/src/test/java/internal/connectors/TestResource.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2015 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.samples.ByteSource; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.util.Stax; -import it.bancaditalia.oss.sdmx.api.DataFlowStructure; -import it.bancaditalia.oss.sdmx.client.Parser; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.util.LanguagePriorityList; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; - -/** - * - * @author Philippe Charles - */ -public final class TestResource { - - @Nonnull - public static final SdmxRepository nbb() { - try { - LanguagePriorityList l = LanguagePriorityList.parse("en"); - SdmxRepository.Builder result = SdmxRepository.builder(); - Map dataStructures = toDataStructures(parse(SdmxSource.NBB_DATA_STRUCTURE, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser())); - result.dataStructures(dataStructures.values()); - DataflowRef flowRef = DataflowRef.of("NBB", "TEST_DATASET", null); - result.dataflows(parse(SdmxSource.NBB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser()) - .stream() - .map(TestResource::toDataFlow) - .filter(o -> o.getRef().equals(flowRef)) - .collect(Collectors.toList())); - DataStructure dsd = dataStructures.get(DataStructureRef.of("NBB", "TEST_DATASET", null)); - try (DataCursor cursor = SdmxXmlStreams.genericData20(dsd).parseReader(XIF, SdmxSource.NBB_DATA::openReader)) { - result.copyOf(flowRef, cursor); - } - return result - .name("NBB") - .seriesKeysOnlySupported(false) - .build(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - @Nonnull - public static final SdmxRepository ecb() { - try { - LanguagePriorityList l = LanguagePriorityList.parse("en"); - SdmxRepository.Builder result = SdmxRepository.builder(); - Map dataStructures = toDataStructures(parse(SdmxSource.ECB_DATA_STRUCTURE, l, new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser())); - result.dataStructures(dataStructures.values()); - DataflowRef flowRef = DataflowRef.of("ECB", "AME", "1.0"); - result.dataflows(parse(SdmxSource.ECB_DATAFLOWS, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()).stream() - .map(Util::toFlow) - .filter(o -> o.getRef().equals(flowRef)) - .collect(Collectors.toList())); - DataStructure dfs = dataStructures.get(DataStructureRef.of("ECB", "ECB_AME1", "1.0")); - try (DataCursor cursor = SdmxXmlStreams.genericData21(dfs).parseReader(XIF, SdmxSource.ECB_DATA::openReader)) { - result.copyOf(flowRef, cursor); - } - return result - .name("ECB") - .seriesKeysOnlySupported(true) - .build(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - private static Map toDataStructures(List input) { - return input.stream() - .map(Util::toStructure) - .collect(Collectors.toMap(DataStructure::getRef, Function.identity())); - } - - private static Dataflow toDataFlow(DataFlowStructure input) { - return Dataflow.of(DataflowRef.of(input.getAgency(), input.getId(), input.getVersion()), - DataStructureRef.of(input.getAgency(), input.getId(), input.getVersion()), - input.getName() - ); - } - - private static T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { - XMLEventReader r = null; - try { - r = XIF.createXMLEventReader(xml.openReader()); - return parser.parse(r, l); - } catch (XMLStreamException | SdmxException ex) { - throw new IOException(ex); - } finally { - if (r != null) { - try { - r.close(); - } catch (XMLStreamException ex) { - throw new RuntimeException(ex); - } - } - } - } - - private static final XMLInputFactory XIF = Stax.getInputFactory(); -} diff --git a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java index 21c0802d..f61eccbf 100644 --- a/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java +++ b/demetra-dotstat-core/src/test/java/internal/sdmx/SdmxQueryUtilTest.java @@ -16,11 +16,8 @@ */ package internal.sdmx; -import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; -import internal.connectors.TestResource; -import be.nbb.sdmx.facade.repo.SdmxRepository; import ec.tss.tsproviders.cursor.TsCursor; import ec.tstoolkit.timeseries.simplets.TsData; import ec.tstoolkit.timeseries.simplets.TsFrequency; @@ -31,6 +28,9 @@ import static internal.sdmx.SdmxQueryUtil.getSeriesWithData; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; +import test.samples.FacadeResource; +import static test.samples.FacadeResource.ECB_FLOW_REF; +import static test.samples.FacadeResource.NBB_FLOW_REF; /** * @@ -38,19 +38,15 @@ */ public class SdmxQueryUtilTest { - private final SdmxRepository nbb = TestResource.nbb(); - private final SdmxRepository ecb = TestResource.ecb(); - private final DataflowRef nbbFlow = DataflowRef.of("NBB", "TEST_DATASET", null); - private final DataflowRef ecbFlow = DataflowRef.parse("ECB,AME,1.0"); private final String title = "FR. Germany - Net lending (+) or net borrowing (-): general government :- Excessive deficit procedure (Including one-off proceeds relative to the allocation of mobile phone licences (UMTS))"; @Test public void testGetAllSeries20() throws Exception { - SdmxConnection conn = nbb.asConnection(); + SdmxConnection conn = FacadeResource.nbb().asConnection(); Key single = Key.of("LOCSTL04", "AUS", "M"); - try (TsCursor o = getAllSeries(conn, nbbFlow, Key.ALL, SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeries(conn, NBB_FLOW_REF, Key.ALL, SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -58,7 +54,7 @@ public void testGetAllSeries20() throws Exception { assertThat(o.nextSeries()).isFalse(); } - try (TsCursor o = getAllSeries(conn, nbbFlow, Key.of("LOCSTL04", "", ""), SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeries(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "", ""), SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -66,7 +62,7 @@ public void testGetAllSeries20() throws Exception { assertThat(o.nextSeries()).isFalse(); } - try (TsCursor o = getAllSeries(conn, nbbFlow, Key.of("LOCSTL04", "AUS", ""), SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeries(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "AUS", ""), SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -77,11 +73,11 @@ public void testGetAllSeries20() throws Exception { @Test public void testGetAllSeriesWithData20() throws Exception { - SdmxConnection conn = nbb.asConnection(); + SdmxConnection conn = FacadeResource.nbb().asConnection(); Key single = Key.of("LOCSTL04", "AUS", "M"); - try (TsCursor o = getAllSeriesWithData(conn, nbbFlow, Key.ALL, SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeriesWithData(conn, NBB_FLOW_REF, Key.ALL, SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -90,7 +86,7 @@ public void testGetAllSeriesWithData20() throws Exception { assertThat(o.nextSeries()).isFalse(); } - try (TsCursor o = getAllSeriesWithData(conn, nbbFlow, Key.of("LOCSTL04", "", ""), SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeriesWithData(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "", ""), SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -99,7 +95,7 @@ public void testGetAllSeriesWithData20() throws Exception { assertThat(o.nextSeries()).isFalse(); } - try (TsCursor o = getAllSeriesWithData(conn, nbbFlow, Key.of("LOCSTL04", "AUS", ""), SdmxQueryUtil.NO_LABEL)) { + try (TsCursor o = getAllSeriesWithData(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "AUS", ""), SdmxQueryUtil.NO_LABEL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesId()).isEqualTo(single); assertThat(o.getSeriesLabel()).isEqualTo(single.toString()); @@ -111,9 +107,9 @@ public void testGetAllSeriesWithData20() throws Exception { @Test public void testGetSeriesWithData20() throws Exception { - SdmxConnection conn = nbb.asConnection(); + SdmxConnection conn = FacadeResource.nbb().asConnection(); - try (TsCursor c = getSeriesWithData(conn, nbbFlow, Key.of("LOCSTL04", "AUS", "M"), SdmxQueryUtil.NO_LABEL)) { + try (TsCursor c = getSeriesWithData(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "AUS", "M"), SdmxQueryUtil.NO_LABEL)) { assertThat(c.nextSeries()).isTrue(); TsData o = c.getSeriesData().get(); assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Monthly, 1966, 1)); @@ -129,21 +125,21 @@ public void testGetSeriesWithData20() throws Exception { @Test public void testGetChildren20() throws Exception { - SdmxConnection conn = nbb.asConnection(); + SdmxConnection conn = FacadeResource.nbb().asConnection(); - assertThat(getChildren(conn, nbbFlow, Key.ALL, 1)).containsExactly("LOCSTL04"); - assertThat(getChildren(conn, nbbFlow, Key.of("LOCSTL04", "", ""), 2)).containsExactly("AUS"); - assertThat(getChildren(conn, nbbFlow, Key.of("LOCSTL04", "AUS", ""), 3)).containsExactly("M"); - assertThat(getChildren(conn, nbbFlow, Key.of("LOCSTL04", "", "M"), 2)).containsExactly("AUS"); + assertThat(getChildren(conn, NBB_FLOW_REF, Key.ALL, 1)).containsExactly("LOCSTL04"); + assertThat(getChildren(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "", ""), 2)).containsExactly("AUS"); + assertThat(getChildren(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "AUS", ""), 3)).containsExactly("M"); + assertThat(getChildren(conn, NBB_FLOW_REF, Key.of("LOCSTL04", "", "M"), 2)).containsExactly("AUS"); } @Test public void testGetAllSeries21() throws Exception { - SdmxConnection conn = ecb.asConnection(); + SdmxConnection conn = FacadeResource.ecb().asConnection(); Key key; key = Key.ALL; - try (TsCursor o = getAllSeries(conn, ecbFlow, key, "EXT_TITLE")) { + try (TsCursor o = getAllSeries(conn, ECB_FLOW_REF, key, "EXT_TITLE")) { int index = 0; while (o.nextSeries()) { switch (index++) { @@ -164,7 +160,7 @@ public void testGetAllSeries21() throws Exception { } key = Key.of("A", "", "", "", "", "", ""); - try (TsCursor o = getAllSeries(conn, ecbFlow, key, "EXT_TITLE")) { + try (TsCursor o = getAllSeries(conn, ECB_FLOW_REF, key, "EXT_TITLE")) { int index = 0; while (o.nextSeries()) { index++; @@ -174,7 +170,7 @@ public void testGetAllSeries21() throws Exception { } key = Key.of("A", "DEU", "", "", "", "", ""); - try (TsCursor o = getAllSeries(conn, ecbFlow, key, "EXT_TITLE")) { + try (TsCursor o = getAllSeries(conn, ECB_FLOW_REF, key, "EXT_TITLE")) { int index = 0; while (o.nextSeries()) { index++; @@ -186,11 +182,11 @@ public void testGetAllSeries21() throws Exception { @Test public void testGetAllSeriesWithData21() throws Exception { - SdmxConnection conn = ecb.asConnection(); + SdmxConnection conn = FacadeResource.ecb().asConnection(); Key key; key = Key.ALL; - try (TsCursor o = getAllSeriesWithData(conn, ecbFlow, key, "EXT_TITLE")) { + try (TsCursor o = getAllSeriesWithData(conn, ECB_FLOW_REF, key, "EXT_TITLE")) { int index = 0; while (o.nextSeries()) { switch (index++) { @@ -207,7 +203,7 @@ public void testGetAllSeriesWithData21() throws Exception { } key = Key.of("A", "DEU", "", "", "", "", ""); - try (TsCursor o = getAllSeriesWithData(conn, ecbFlow, key, "EXT_TITLE")) { + try (TsCursor o = getAllSeriesWithData(conn, ECB_FLOW_REF, key, "EXT_TITLE")) { int index = 0; while (o.nextSeries()) { switch (index++) { @@ -226,11 +222,11 @@ public void testGetAllSeriesWithData21() throws Exception { @Test public void testGetSeriesWithData21() throws Exception { - SdmxConnection conn = ecb.asConnection(); + SdmxConnection conn = FacadeResource.ecb().asConnection(); Key key = Key.of("A", "DEU", "1", "0", "319", "0", "UBLGE"); - try (TsCursor c = getSeriesWithData(conn, ecbFlow, key, SdmxQueryUtil.NO_LABEL)) { + try (TsCursor c = getSeriesWithData(conn, ECB_FLOW_REF, key, SdmxQueryUtil.NO_LABEL)) { assertThat(c.nextSeries()).isTrue(); TsData o = c.getSeriesData().get(); assertThat(o.getStart()).isEqualTo(new TsPeriod(TsFrequency.Yearly, 1991, 0)); @@ -245,11 +241,11 @@ public void testGetSeriesWithData21() throws Exception { @Test public void testGetChildren21() throws Exception { - SdmxConnection conn = ecb.asConnection(); + SdmxConnection conn = FacadeResource.ecb().asConnection(); - assertThat(getChildren(conn, ecbFlow, Key.ALL, 1)).containsExactly("A"); - assertThat(getChildren(conn, ecbFlow, Key.of("A", "", "", "", "", "", ""), 2)).hasSize(30).contains("BEL", "POL"); - assertThat(getChildren(conn, ecbFlow, Key.of("A", "BEL", "", "", "", "", ""), 3)).containsExactly("1"); - assertThat(getChildren(conn, ecbFlow, Key.of("hello", "", "", "", "", "", ""), 2)).isEmpty(); + assertThat(getChildren(conn, ECB_FLOW_REF, Key.ALL, 1)).containsExactly("A"); + assertThat(getChildren(conn, ECB_FLOW_REF, Key.of("A", "", "", "", "", "", ""), 2)).hasSize(30).contains("BEL", "POL"); + assertThat(getChildren(conn, ECB_FLOW_REF, Key.of("A", "BEL", "", "", "", "", ""), 3)).containsExactly("1"); + assertThat(getChildren(conn, ECB_FLOW_REF, Key.of("hello", "", "", "", "", "", ""), 2)).isEmpty(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java similarity index 92% rename from sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java rename to demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index df8a15e8..afa8531a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package test; +package test.samples; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -29,7 +29,6 @@ import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import be.nbb.util.Stax; -import internal.connectors.Util; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -46,6 +45,8 @@ public class FacadeResource { public static final DataflowRef ECB_FLOW_REF = DataflowRef.of("ECB", "AME", "1.0"); public static final DataStructureRef ECB_STRUCT_REF = DataStructureRef.of("ECB", "ECB_AME1", "1.0"); + public static final DataflowRef NBB_FLOW_REF = DataflowRef.of("NBB", "TEST_DATASET", null); + public SdmxRepository nbb() throws IOException { SdmxRepository result = NBB.get(); if (result == null) { @@ -55,12 +56,10 @@ public SdmxRepository nbb() throws IOException { List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); - DataflowRef ref = DataflowRef.of("NBB", "TEST_DATASET", null); - result = SdmxRepository.builder() .dataStructures(structs) .dataflows(flows) - .data(ref, data) + .data(NBB_FLOW_REF, data) .name("NBB") .seriesKeysOnlySupported(false) .build(); @@ -100,6 +99,7 @@ private List struct20(XMLInputFactory f, ByteSource xml, Language } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + // FIXME: find sample of dataflow20 ? return struct20(f, xml, l).stream() .map(FacadeResource::asDataflow) .collect(Collectors.toList()); @@ -116,11 +116,7 @@ private List struct21(XMLInputFactory f, ByteSource xml, Language } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - // FIXME: no facade impl yet - return ConnectorsResource.parse(xml, Util.fromLanguages(l), new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()) - .stream() - .map(Util::toFlow) - .collect(Collectors.toList()); + return SdmxXmlStreams.flow21(l).parseReader(f, xml::openReader); } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index c8125734..7da6c466 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -20,6 +20,7 @@ import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.parser.ObsParser; @@ -83,4 +84,9 @@ public Stax.Parser> struct20(@Nonnull LanguagePriorityList l public Stax.Parser> struct21(@Nonnull LanguagePriorityList langs) throws IOException { return Stax.Parser.of(new XMLStreamStructure21(langs)::parse); } + + @Nonnull + public Stax.Parser> flow21(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.Parser.of(new XMLStreamFlow21(langs)::parse); + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java new file mode 100644 index 00000000..bb63f348 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java @@ -0,0 +1,144 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import be.nbb.util.Stax; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import java.util.ArrayList; +import java.util.List; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.*; +import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * + * @author Philippe Charles + */ +@NotThreadSafe +final class XMLStreamFlow21 { + + private static final String HEADER_TAG = "Header"; + private static final String STRUCTURES_TAG = "Structures"; + private static final String DATAFLOWS_TAG = "Dataflows"; + private static final String DATAFLOW_TAG = "Dataflow"; + private static final String NAME_TAG = "Name"; + private static final String STRUCTURE_TAG = "Structure"; + private static final String REF_TAG = "Ref"; + + private static final String ID_ATTR = "id"; + private static final String AGENCY_ID_ATTR = "agencyID"; + private static final String VERSION_ATTR = "version"; + private static final String LANG_ATTR = "lang"; + + private final TextBuilder flowLabel; + + XMLStreamFlow21(LanguagePriorityList languages) { + this.flowLabel = new TextBuilder(languages); + } + + @Nonnull + public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { + if (Stax.isNotNamespaceAware(reader)) { + throw new XMLStreamException("Cannot parse flows"); + } + + List result = new ArrayList<>(); + while (nextTags(reader, "")) { + switch (reader.getLocalName()) { + case HEADER_TAG: + parseHeader(reader); + break; + case STRUCTURES_TAG: + parseStructures(reader, result); + break; + } + } + return result; + } + + private void parseHeader(XMLStreamReader reader) throws XMLStreamException { + String ns = reader.getNamespaceURI(); + check(NS_V21_URI.equals(ns), reader, "Invalid namespace '%s'", ns); + } + + private void parseStructures(XMLStreamReader reader, List flows) throws XMLStreamException { + if (nextTag(reader, STRUCTURES_TAG, DATAFLOWS_TAG)) { + parseDataflows(reader, flows); + } + } + + private void parseDataflows(XMLStreamReader reader, List flows) throws XMLStreamException { + while (nextTags(reader, DATAFLOWS_TAG)) { + switch (reader.getLocalName()) { + case DATAFLOW_TAG: + flows.add(parseDataflow(reader)); + break; + } + } + } + + @SuppressWarnings("null") + private Dataflow parseDataflow(XMLStreamReader reader) throws XMLStreamException { + String id = reader.getAttributeValue(null, ID_ATTR); + check(id != null, reader, "Missing Dataflow id"); + + DataflowRef flowRef = DataflowRef.of(reader.getAttributeValue(null, AGENCY_ID_ATTR), id, reader.getAttributeValue(null, VERSION_ATTR)); + DataStructureRef structRef = null; + flowLabel.clear(); + while (nextTags(reader, DATAFLOW_TAG)) { + switch (reader.getLocalName()) { + case NAME_TAG: + parseNameTag(reader, flowLabel); + break; + case STRUCTURE_TAG: + structRef = parseStructure(reader); + break; + } + } + + check(structRef != null, reader, "Missing DataStructureRef"); + + return Dataflow.of(flowRef, structRef, flowLabel.build(id)); + } + + private DataStructureRef parseStructure(XMLStreamReader reader) throws XMLStreamException { + if (nextTag(reader, STRUCTURE_TAG, REF_TAG)) { + return parseRef(reader); + } + throw new XMLStreamException("Missing DataStructureRef"); + } + + private DataStructureRef parseRef(XMLStreamReader reader) throws XMLStreamException { + String id = reader.getAttributeValue(null, ID_ATTR); + check(id != null, reader, "Missing DataStructureRef id"); + + return DataStructureRef.of(reader.getAttributeValue(null, AGENCY_ID_ATTR), id, reader.getAttributeValue(null, VERSION_ATTR)); + } + + private void parseNameTag(XMLStreamReader reader, TextBuilder langStack) throws XMLStreamException { + String lang = reader.getAttributeValue(null, LANG_ATTR); + if (lang != null) { + langStack.put(lang, reader.getElementText()); + } + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java new file mode 100644 index 00000000..9f806cce --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.xml.stream; + +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.util.Stax; +import java.io.IOException; +import java.util.List; +import javax.xml.stream.XMLInputFactory; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class XMLStreamFlow21Test { + + @Test + public void test() throws IOException { + Stax.Parser> p = Stax.Parser.of(new XMLStreamFlow21(LanguagePriorityList.ANY)::parse); + + assertThat(p.parseReader(xif, SdmxSource.ECB_DATAFLOWS::openReader)) + .containsExactly( + Dataflow.of(DataflowRef.of("ECB", "AME", "1.0"), DataStructureRef.of("ECB", "ECB_AME1", "1.0"), "AMECO"), + Dataflow.of(DataflowRef.of("ECB", "BKN", "1.0"), DataStructureRef.of("ECB", "ECB_BKN1", "1.0"), "Banknotes statistics"), + Dataflow.of(DataflowRef.of("ECB", "BLS", "1.0"), DataStructureRef.of("ECB", "ECB_BLS1", "1.0"), "Bank Lending Survey Statistics") + ); + } + + private final XMLInputFactory xif = Stax.getInputFactory(); +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index 6b9c191c..cfeb4a24 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -16,7 +16,7 @@ */ package internal.connectors; -import test.ConnectorsResource; +import test.samples.ConnectorsResource; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.samples.SdmxSource; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java index 932352eb..22404d5d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java @@ -29,9 +29,9 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import test.FacadeResource; -import static test.FacadeResource.ECB_FLOW_REF; -import static test.FacadeResource.ECB_STRUCT_REF; +import test.samples.FacadeResource; +import static test.samples.FacadeResource.ECB_FLOW_REF; +import static test.samples.FacadeResource.ECB_STRUCT_REF; import test.client.RepoRestClient; /** diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java index ad105329..2cef17b6 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java @@ -24,9 +24,9 @@ import java.io.IOException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import test.FacadeResource; -import static test.FacadeResource.ECB_FLOW_REF; -import static test.FacadeResource.ECB_STRUCT_REF; +import test.samples.FacadeResource; +import static test.samples.FacadeResource.ECB_FLOW_REF; +import static test.samples.FacadeResource.ECB_STRUCT_REF; import test.client.FailingRestClient; import test.client.NoOpRestClient; import test.client.NullRestClient; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java index 5a114831..398ed95e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/RestConnectionTest.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.tck.ConnectionAssert; import java.io.IOException; import org.junit.Test; -import test.FacadeResource; +import test.samples.FacadeResource; import test.client.RepoRestClient; /** diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java similarity index 87% rename from sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java index 50e2818c..1cd1584a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package test; +package test.samples; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Frequency; @@ -90,21 +90,21 @@ public SdmxRepository ecb() throws IOException { .build(); } - DataflowRef firstOf(List flows) { + private DataflowRef firstOf(List flows) { return flows.stream().map(o -> Util.toFlow(o).getRef()).findFirst().get(); } - List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { + private List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v20.DataStructureParser()); } - List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { + private List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { return struct20(xml, l).stream() .map(ConnectorsResource::asDataflow) .collect(Collectors.toList()); } - List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { + private List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl return FacadeResource.data20(XIF, xml, Util.toStructure(dsd)) .stream() @@ -116,7 +116,7 @@ public List struct21(ByteSource xml, LanguagePriorityList l) return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataStructureParser()); } - List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { + private List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { return parse(xml, l, new it.bancaditalia.oss.sdmx.parser.v21.DataflowParser()); } @@ -128,7 +128,7 @@ public List data21(ByteSource xml, DataFlowStructure dsd, La .collect(Collectors.toList()); } - PortableTimeSeries toPortableTimeSeries(Series o, List dims) { + private PortableTimeSeries toPortableTimeSeries(Series o, List dims) { PortableTimeSeries result = new PortableTimeSeries(); result.setFrequency(String.valueOf(formatByStandardFreq(o.getFreq()))); o.getMeta().forEach(result::addAttribute); @@ -140,11 +140,11 @@ PortableTimeSeries toPortableTimeSeries(Series o, List dims) { return result; } - String valueToString(Double o) { + private String valueToString(Double o) { return o != null ? o.toString() : ""; } - String periodToString(Frequency f, LocalDateTime o) { + private String periodToString(Frequency f, LocalDateTime o) { if (o == null) { return ""; } @@ -158,7 +158,7 @@ String periodToString(Frequency f, LocalDateTime o) { } } - Dataflow asDataflow(DataFlowStructure o) { + private Dataflow asDataflow(DataFlowStructure o) { Dataflow result = new Dataflow(); result.setAgency(o.getAgency()); result.setDsdIdentifier(new DSDIdentifier(o.getId(), o.getAgency(), o.getVersion())); @@ -168,7 +168,7 @@ Dataflow asDataflow(DataFlowStructure o) { return result; } - T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { + private T parse(ByteSource xml, LanguagePriorityList l, Parser parser) throws IOException { XMLEventReader r = null; try { r = XIF.createXMLEventReader(xml.openReader()); @@ -210,6 +210,6 @@ private char formatByStandardFreq(Frequency code) { return '?'; } } - + private final XMLInputFactory XIF = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java new file mode 100644 index 00000000..afa8531a --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java @@ -0,0 +1,134 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package test.samples; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.samples.ByteSource; +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.util.SeriesSupport; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import be.nbb.util.Stax; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import javax.xml.stream.XMLInputFactory; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class FacadeResource { + + public static final DataflowRef ECB_FLOW_REF = DataflowRef.of("ECB", "AME", "1.0"); + public static final DataStructureRef ECB_STRUCT_REF = DataStructureRef.of("ECB", "ECB_AME1", "1.0"); + + public static final DataflowRef NBB_FLOW_REF = DataflowRef.of("NBB", "TEST_DATASET", null); + + public SdmxRepository nbb() throws IOException { + SdmxRepository result = NBB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); + + result = SdmxRepository.builder() + .dataStructures(structs) + .dataflows(flows) + .data(NBB_FLOW_REF, data) + .name("NBB") + .seriesKeysOnlySupported(false) + .build(); + + NBB.set(result); + } + return result; + } + + public SdmxRepository ecb() throws IOException { + SdmxRepository result = ECB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); + List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); + + result = SdmxRepository.builder() + .dataStructures(structs) + .dataflows(flows) + .data(ECB_FLOW_REF, data) + .name("ECB") + .seriesKeysOnlySupported(true) + .build(); + + ECB.set(result); + } + return result; + } + + private static final AtomicReference NBB = new AtomicReference<>(); + private static final AtomicReference ECB = new AtomicReference<>(); + + private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct20(l).parseReader(f, xml::openReader); + } + + private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + // FIXME: find sample of dataflow20 ? + return struct20(f, xml, l).stream() + .map(FacadeResource::asDataflow) + .collect(Collectors.toList()); + } + + List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(f, xml::openReader)) { + return SeriesSupport.copyOf(c); + } + } + + private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct21(l).parseReader(f, xml::openReader); + } + + private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.flow21(l).parseReader(f, xml::openReader); + } + + List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(f, xml::openReader)) { + return SeriesSupport.copyOf(c); + } + } + + private Dataflow asDataflow(DataStructure o) { + DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); + return Dataflow.of(ref, o.getRef(), o.getLabel()); + } + + private final XMLInputFactory XIF = Stax.getInputFactory(); +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ParsersTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java rename to sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ParsersTest.java index dcd65693..cd45f71c 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/ParsersTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ParsersTest.java @@ -14,7 +14,7 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package test; +package test.samples; import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; From d0149dffec2d1d5e1a13ba13e10d3dd2c78ec531 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 30 Nov 2017 12:58:07 +0100 Subject: [PATCH 57/68] Added initial native implementation of SDMX21 driver. --- .../internal/file/SdmxFileConnectionImpl.java | 3 +- .../nbb/sdmx/facade/repo/SdmxRepository.java | 14 +- .../nbb/sdmx/facade/util/SdmxExceptions.java | 50 ++++++ .../src/main/java/be/nbb/util/IO.java | 6 + .../src/main/java/be/nbb/util/Stax.java | 7 +- .../nbb/sdmx/facade/web/SdmxWebManager.java | 5 + .../connectors/ConnectorRestClient.java | 2 +- .../java/internal/web/CachedRestClient.java | 6 +- .../java/internal/web/FailsafeRestClient.java | 4 +- .../main/java/internal/web/RestClient.java | 2 +- .../java/internal/web/RestConnection.java | 5 +- .../internal/web/sdmx21/RestExecutor.java | 38 +++++ .../internal/web/sdmx21/RestExecutorImpl.java | 151 ++++++++++++++++++ .../internal/web/sdmx21/Sdmx21Driver2.java | 69 ++++++++ .../internal/web/sdmx21/Sdmx21RestClient.java | 107 +++++++++++++ .../web/sdmx21/Sdmx21RestQueries.java | 82 ++++++++++ .../java/internal/web/sdmx21/URLBuilder.java | 90 +++++++++++ .../internal/web/CachedRestClientTest.java | 10 +- .../internal/web/FailsafeRestClientTest.java | 2 +- .../java/test/client/CallStackClient.java | 4 +- .../java/test/client/FailingRestClient.java | 2 +- .../test/java/test/client/NoOpRestClient.java | 2 +- .../test/java/test/client/NullRestClient.java | 2 +- .../test/java/test/client/RepoRestClient.java | 9 +- 24 files changed, 640 insertions(+), 32 deletions(-) create mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutor.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21Driver2.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestQueries.java create mode 100644 sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/URLBuilder.java diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java index 076a3251..6ebdc92e 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java @@ -25,6 +25,7 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileConnection; +import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; import java.util.Collections; @@ -125,7 +126,7 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw new IOException("Connection closed"); + throw SdmxExceptions.connectionClosed(); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index a61db293..24114f51 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -24,6 +24,7 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; import java.util.ArrayList; @@ -163,15 +164,16 @@ public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); return repo .getFlow(flowRef) - .orElseThrow(() -> new IOException("Dataflow not found")); + .orElseThrow(() -> SdmxExceptions.missingFlow(flowRef)); } @Override public DataStructure getStructure(DataflowRef flowRef) throws IOException { checkState(); + DataStructureRef structRef = getFlow(flowRef).getStructureRef(); return repo - .getStructure(getFlow(flowRef).getStructureRef()) - .orElseThrow(() -> new IOException("DataStructure not found")); + .getStructure(structRef) + .orElseThrow(() -> SdmxExceptions.missingStructure(structRef)); } @Override @@ -179,7 +181,7 @@ public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOExcep checkState(); return repo .getCursor(flowRef, query) - .orElseThrow(() -> new IOException("Data not found")); + .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); } @Override @@ -187,7 +189,7 @@ public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOE checkState(); return repo .getStream(flowRef, query) - .orElseThrow(() -> new IOException("Data not found")); + .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); } @Override @@ -202,7 +204,7 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw new IOException("Connection closed"); + throw SdmxExceptions.connectionClosed(); } } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java new file mode 100644 index 00000000..da986017 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.util; + +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.DataflowRef; +import java.io.IOException; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public final class SdmxExceptions { + + @Nonnull + public IOException connectionClosed() { + return new IOException("Connection already closed"); + } + + @Nonnull + public IOException missingFlow(@Nonnull DataflowRef ref) { + return new IOException("Missing dataflow '" + ref + "'"); + } + + @Nonnull + public IOException missingStructure(@Nonnull DataStructureRef ref) { + return new IOException("Missing datastructure '" + ref + "'"); + } + + @Nonnull + public IOException missingData(@Nonnull DataflowRef ref) { + return new IOException("Missing data '" + ref + "'"); + } +} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java index 665deb0a..35229c9a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java @@ -190,6 +190,12 @@ static Consumer noOp() { } } + @FunctionalInterface + public interface Parser { + + R parseWithIO(@Nonnull Supplier input) throws IOException; + } + @Nonnull public Stream stream(@Nonnull Supplier supplier, @Nonnull Function> stream) throws IOException { T resource = supplier.getWithIO(); diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java index 571fa87e..c27d199d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java @@ -59,6 +59,9 @@ public interface Function { @FunctionalInterface public interface Parser { + @Nonnull + T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; + @Nonnull default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull File file, @Nonnull Charset cs) throws IOException { return parseStream(xf, () -> new FileInputStream(file), cs); @@ -97,7 +100,9 @@ default T parse(@Nonnull Supplier supplier, @Nonnull } @Nonnull - T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; + default IO.Parser asStreamParser(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + return o -> parseStream(xf, o, cs); + } @Nonnull static Parser of(@Nonnull Function func) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 8794b9b3..e31a3e1e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -84,6 +84,11 @@ public SdmxConnection getConnection(String name, LanguagePriorityList languages) return getConnection(entryPoint, languages); } + @Nonnull + public SdmxConnection getConnection(@Nonnull SdmxWebEntryPoint entryPoint) throws IOException { + return getConnection(entryPoint, LanguagePriorityList.ANY); + } + @Nonnull public SdmxConnection getConnection(@Nonnull SdmxWebEntryPoint entryPoint, @Nonnull LanguagePriorityList languages) throws IOException { Objects.requireNonNull(entryPoint); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java index 59ecb2a0..4a9c559a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java @@ -118,7 +118,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { try { return connector instanceof HasDataCursor ? getCursor((HasDataCursor) connector, flowRef, dsd, query) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java index 1a579185..9a25344a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedRestClient.java @@ -76,9 +76,9 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { if (!query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)) { - return delegate.getData(flowRef, dsd, query); + return delegate.getData(flowRef, query, dsd); } return loadKeysOnlyWithCache(flowRef, dsd, query).getCursor(flowRef, query) .orElseThrow(() -> new IOException("Data not found")); @@ -153,7 +153,7 @@ private boolean isBroaderRequest(Key key, SdmxRepository repo) { } private SdmxRepository copyDataKeys(DataflowRef flowRef, DataStructure structure, DataQuery query) throws IOException { - try (DataCursor cursor = delegate.getData(flowRef, structure, query)) { + try (DataCursor cursor = delegate.getData(flowRef, query, structure)) { return SdmxRepository.builder() .copyOf(flowRef, cursor) .name(query.getKey().toString()) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java index 0a7709c8..596cee47 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java @@ -90,11 +90,11 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { DataCursor result; try { - result = delegate.getData(flowRef, dsd, query); + result = delegate.getData(flowRef, query, dsd); } catch (RuntimeException ex) { throw unexpected(ex, "Unexpected exception while getting data from dataset '%s' with key '%s'", flowRef, query.getKey()); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java index a8f6a656..2a859c72 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestClient.java @@ -45,7 +45,7 @@ public interface RestClient { DataStructure getStructure(@Nonnull DataStructureRef ref) throws IOException; @Nonnull - DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataStructure dsd, @Nonnull DataQuery query) throws IOException; + DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query, @Nonnull DataStructure dsd) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java index d2eac4b0..1ee60226 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/RestConnection.java @@ -25,6 +25,7 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; import java.util.HashSet; @@ -80,7 +81,7 @@ public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOExcep } DataStructure structure = client.getStructure(structRef); - return client.getData(flowRef, structure, query); + return client.getData(flowRef, query, structure); } @Override @@ -100,7 +101,7 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw new IOException("Connection closed"); + throw SdmxExceptions.connectionClosed(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutor.java new file mode 100644 index 00000000..61dda029 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutor.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +@FunctionalInterface +public interface RestExecutor { + + @Nonnull + InputStream execute(@Nonnull URL query, @Nonnull String mediaType, @Nonnull LanguagePriorityList langs) throws IOException; + + static RestExecutor getDefault(int readTimeout, int connectTimeout) { + return RestExecutorImpl.of(readTimeout, connectTimeout); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java new file mode 100644 index 00000000..feb33f30 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java @@ -0,0 +1,151 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.util.IO; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; +import javax.annotation.Nullable; + +/** + * + * @author Philippe Charles + */ +@lombok.RequiredArgsConstructor(staticName = "of") +final class RestExecutorImpl implements RestExecutor { + + private static final String ACCEPT_HEADER = "Accept"; + private static final String ACCEPT_LANGUAGE_HEADER = "Accept-Language"; + private static final String ACCEPT_ENCODING_HEADER = "Accept-Encoding"; + private static final String LOCATION_HEADER = "Location"; + + private final int readTimeout; + private final int connectTimeout; + private final int maxHop = 3; + + @Override + public InputStream execute(URL query, String mediaType, LanguagePriorityList langs) throws IOException { + return call(query, mediaType, langs, 0); + } + + private InputStream call(URL query, String mediaType, LanguagePriorityList langs, int hop) throws IOException { + URLConnection conn = query.openConnection(); + conn.setReadTimeout(readTimeout); + conn.setConnectTimeout(connectTimeout); + + if (!(conn instanceof HttpURLConnection)) { + throw new IOException("Unsupported connection type"); + } + + HttpURLConnection http = (HttpURLConnection) conn; + http.setRequestMethod("GET"); + http.setRequestProperty(ACCEPT_HEADER, mediaType); + http.setRequestProperty(ACCEPT_LANGUAGE_HEADER, langs.toString()); + http.addRequestProperty(ACCEPT_ENCODING_HEADER, ContentEncoder.getEncodingHeader()); + + switch (http.getResponseCode()) { + case HttpURLConnection.HTTP_MULT_CHOICE: + case HttpURLConnection.HTTP_SEE_OTHER: + return redirect(http, mediaType, langs, hop); + case HttpURLConnection.HTTP_OK: + return getBody(http); + default: + throw getError(http); + } + } + + private InputStream redirect(HttpURLConnection http, String mediaType, LanguagePriorityList langs, int hop) throws IOException { + try { + if (hop == maxHop) { + throw new IOException("Max hop reached"); + } + String newQuery = http.getHeaderField(LOCATION_HEADER); + if (newQuery == null || newQuery.isEmpty()) { + throw new IOException("Missing redirection url"); + } + return call(new URL(newQuery), mediaType, langs, hop + 1); + } finally { + http.disconnect(); + } + } + + private InputStream getBody(HttpURLConnection http) throws IOException { + String encoding = http.getContentEncoding(); + InputStream body = ContentEncoder.getDecoder(encoding).applyWithIO(http.getInputStream()); + return new DisconnectingInputStream(body, http::disconnect); + } + + private IOException getError(HttpURLConnection http) throws IOException { + try { + System.out.println(http.getURL()); + return new IOException(http.getResponseCode() + ": " + http.getResponseMessage()); + } finally { + http.disconnect(); + } + } + + @lombok.RequiredArgsConstructor + private static final class DisconnectingInputStream extends InputStream { + + @lombok.experimental.Delegate(excludes = Closeable.class) + private final InputStream delegate; + + private final Runnable onClose; + + @Override + public void close() throws IOException { + try { + delegate.close(); + } finally { + onClose.run(); + } + } + } + + @lombok.RequiredArgsConstructor + @lombok.Getter + private enum ContentEncoder { + + GZIP("gzip", GZIPInputStream::new), + DEFLATE("deflate", InflaterInputStream::new); + + private final String name; + private final IO.Function decoder; + + static IO.Function getDecoder(@Nullable String name) { + return Stream.of(values()) + .filter(o -> Objects.equals(name, o.name)) + .map(o -> o.getDecoder()) + .findAny() + .orElse(o -> o); + } + + static String getEncodingHeader() { + return Stream.of(ContentEncoder.values()).map(ContentEncoder::getName).collect(Collectors.joining(",")); + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21Driver2.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21Driver2.java new file mode 100644 index 00000000..b2de88e6 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21Driver2.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.util.CommonSdmxProperty; +import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.connectors.Util; +import internal.web.RestClient; +import internal.web.RestDriverSupport; +import java.net.MalformedURLException; +import java.net.URL; +import javax.annotation.Nonnull; + +/** + * + * @author Philippe Charles + */ +public final class Sdmx21Driver2 implements SdmxWebDriver { + + private static final String PREFIX = "sdmx:sdmx21:"; + + @lombok.experimental.Delegate + private final RestDriverSupport support = RestDriverSupport + .builder() + .prefix(PREFIX) + .client(Sdmx21Driver2::of) + .build(); + + private static RestClient of(SdmxWebEntryPoint o, String prefix, @Nonnull LanguagePriorityList languages) { + return Sdmx21RestClient.of(getEndPoint(o, prefix), isSeriesKeysOnly(o), languages, getExecutor(o)); + } + + private static URL getEndPoint(SdmxWebEntryPoint o, String prefix) { + try { + return new URL(o.getUri().toString().substring(prefix.length())); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } + + private static boolean isSeriesKeysOnly(SdmxWebEntryPoint o) { + return Util.SERIES_KEYS_ONLY_SUPPORTED.get(o.getProperties(), false); + } + + private static RestExecutor getExecutor(SdmxWebEntryPoint o) { + return RestExecutor.getDefault( + CommonSdmxProperty.READ_TIMEOUT.get(o.getProperties(), DEFAULT_READ_TIMEOUT), + CommonSdmxProperty.CONNECT_TIMEOUT.get(o.getProperties(), DEFAULT_CONNECT_TIMEOUT)); + } + + private final static int DEFAULT_CONNECT_TIMEOUT = 1000 * 60 * 2; // 2 minutes + private final static int DEFAULT_READ_TIMEOUT = 1000 * 60 * 2; // 2 minutes +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java new file mode 100644 index 00000000..45a45b59 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java @@ -0,0 +1,107 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.parser.DataFactory; +import be.nbb.sdmx.facade.util.SdmxExceptions; +import static be.nbb.sdmx.facade.util.SdmxMediaType.*; +import static be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams.*; +import be.nbb.util.IO; +import be.nbb.util.Stax; +import internal.web.RestClient; +import static internal.web.sdmx21.Sdmx21RestQueries.*; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import static java.nio.charset.StandardCharsets.UTF_8; +import java.util.List; + +/** + * + * @author Philippe Charles + */ +@lombok.RequiredArgsConstructor(staticName = "of") +public final class Sdmx21RestClient implements RestClient { + + private final URL endpoint; + private final boolean seriesKeysOnlySupported; + private final LanguagePriorityList langs; + private final RestExecutor executor; + + private final DataFactory dialect = DataFactory.sdmx21(); + + @Override + public List getFlows() throws IOException { + URL url = getFlowsQuery(endpoint); + return flow21(langs) + .asStreamParser(Stax.getInputFactory(), UTF_8) + .parseWithIO(calling(url, XML)); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + URL url = getFlowQuery(endpoint, ref); + return flow21(langs) + .asStreamParser(Stax.getInputFactory(), UTF_8) + .parseWithIO(calling(url, XML)) + .stream() + .filter(o -> ref.contains(o.getRef())) + .findFirst() + .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + URL url = getStructureQuery(endpoint, ref); + return struct21(langs) + .asStreamParser(Stax.getInputFactory(), UTF_8) + .parseWithIO(calling(url, STRUCTURE_21)) + .stream() + .filter(o -> ref.equals(o.getRef())) + .findFirst() + .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); + } + + @Override + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { + URL url = getDataQuery(endpoint, flowRef, query); + return compactData21(dsd, dialect) + .asStreamParser(Stax.getInputFactoryWithoutNamespace(), UTF_8) + .parseWithIO(calling(url, STRUCTURE_SPECIFIC_DATA_21)); + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + return seriesKeysOnlySupported; + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + return null; + } + + private IO.Supplier calling(URL query, String mediaType) throws IOException { + return () -> executor.execute(query, mediaType, langs); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestQueries.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestQueries.java new file mode 100644 index 00000000..3d6eb571 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestQueries.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; +import be.nbb.sdmx.facade.ResourceRef; +import java.io.IOException; +import java.net.URL; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +class Sdmx21RestQueries { + + URLBuilder onMeta(URL endpoint, String resourceType, ResourceRef ref) { + return URLBuilder.of(endpoint) + .path(resourceType) + .path(ref.getAgency()) + .path(ref.getId()) + .path(ref.getVersion()); + } + + URLBuilder onData(URL endpoint, DataflowRef flowRef, Key key) throws IOException { + return URLBuilder.of(endpoint) + .path(DATA_RESOURCE) + .path(flowRef.toString()) + .path(key.toString()) + .path(DEFAULT_PROVIDER_REF); + } + + URL getFlowsQuery(URL endpoint) throws IOException { + return onMeta(endpoint, DATAFLOW_RESOURCE, FLOWS).build(); + } + + URL getFlowQuery(URL endpoint, DataflowRef ref) throws IOException { + return onMeta(endpoint, DATAFLOW_RESOURCE, ref).build(); + } + + URL getStructureQuery(URL endpoint, DataStructureRef ref) throws IOException { + return onMeta(endpoint, DATASTRUCTURE_RESOURCE, ref).param(REFERENCES_PARAM, "children").build(); + } + + URL getDataQuery(URL endpoint, DataflowRef flowRef, DataQuery query) throws IOException { + URLBuilder result = onData(endpoint, flowRef, query.getKey()); + switch (query.getDetail()) { + case SERIES_KEYS_ONLY: + result.param(DETAIL_PARAM, "serieskeysonly"); + break; + } + return result.build(); + } + + private static final DataflowRef FLOWS = DataflowRef.of("all", "all", "latest"); + + private static final String DATAFLOW_RESOURCE = "dataflow"; + private static final String DATASTRUCTURE_RESOURCE = "datastructure"; + private static final String DATA_RESOURCE = "data"; + + private static final String DEFAULT_PROVIDER_REF = "all"; + + private static final String REFERENCES_PARAM = "references"; + private static final String DETAIL_PARAM = "detail"; +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/URLBuilder.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/URLBuilder.java new file mode 100644 index 00000000..451ebf62 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/URLBuilder.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.sdmx21; + +import java.io.IOException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * + * @author Philippe Charles + */ +@lombok.RequiredArgsConstructor(staticName = "of") +final class URLBuilder { + + private final URL entryPoint; + private final List paths = new ArrayList<>(); + private final Map params = new LinkedHashMap<>(); + + /** + * Appends the specified path the current URL. + * + * @param path a non-null path + * @return this builder + * @throws NullPointerException if entryPoint is null + */ + public URLBuilder path(String path) { + Objects.requireNonNull(path); + paths.add(path); + return this; + } + + /** + * Appends the specified parameter to the current URL. + * + * @param key a non-null key + * @param value a non-null value + * @return this builder + * @throws NullPointerException if key or value is null + */ + public URLBuilder param(String key, String value) { + Objects.requireNonNull(key); + Objects.requireNonNull(value); + params.put(key, value); + return this; + } + + /** + * Creates a new URL using the specified path and parameters. + * + * @return a URL + * @throws IOException + */ + public URL build() throws IOException { + StringBuilder result = new StringBuilder(); + result.append(entryPoint); + + for (String path : paths) { + result.append('/').append(URLEncoder.encode(path, "UTF-8")); + } + + boolean first = true; + for (Map.Entry entry : params.entrySet()) { + result.append(first ? '?' : '&'); + result.append(URLEncoder.encode(entry.getKey(), "UTF-8")).append('=').append(entry.getValue()); + first = false; + } + + return new URL(result.toString()); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java index 22404d5d..ac4b7080 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedRestClientTest.java @@ -139,23 +139,23 @@ public void testLoadData() throws IOException { CachedRestClient target = new CachedRestClient(getClient(count), "", cache, clock, 100); - assertThatNullPointerException().isThrownBy(() -> target.getData(null, null, query)); + assertThatNullPointerException().isThrownBy(() -> target.getData(null, query, null)); - target.getData(ECB_FLOW_REF, null, query); + target.getData(ECB_FLOW_REF, query, null); assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); - target.getData(ECB_FLOW_REF, null, query); + target.getData(ECB_FLOW_REF, query, null); assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); clock.plus(100); - target.getData(ECB_FLOW_REF, null, query); + target.getData(ECB_FLOW_REF, query, null); assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(keysId); cache.clear(); - target.getData(ECB_FLOW_REF, null, query); + target.getData(ECB_FLOW_REF, query, null); assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(keysId); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java index 2cef17b6..e46fa348 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java @@ -58,7 +58,7 @@ public void testGetData() { DataQuery all = DataQuery.of(Key.ALL, false); DataStructure struct = DataStructure.builder().ref(ECB_STRUCT_REF).label("hello").build(); - assertOperation(o -> FailsafeRestClient.of(o).getData(ECB_FLOW_REF, struct, all)); + assertOperation(o -> FailsafeRestClient.of(o).getData(ECB_FLOW_REF, all, struct)); } @Test diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java index baf34c87..de938e6d 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/CallStackClient.java @@ -59,9 +59,9 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { count.incrementAndGet(); - return delegate.getData(flowRef, dsd, query); + return delegate.getData(flowRef, query, dsd); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java index 53da214d..97d124cf 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/FailingRestClient.java @@ -50,7 +50,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java index a2ea9165..069b64d5 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NoOpRestClient.java @@ -53,7 +53,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { Objects.requireNonNull(flowRef); Objects.requireNonNull(dsd); Objects.requireNonNull(query); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java index 8b745f8b..f130f1ae 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/NullRestClient.java @@ -53,7 +53,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { Objects.requireNonNull(flowRef); Objects.requireNonNull(dsd); Objects.requireNonNull(query); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java index 0664fc20..32dd3e8e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java @@ -23,6 +23,7 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.util.SdmxExceptions; import internal.web.RestClient; import java.io.IOException; import java.util.ArrayList; @@ -46,19 +47,19 @@ public List getFlows() throws IOException { @Override public Dataflow getFlow(DataflowRef ref) throws IOException { return repo.getFlow(ref) - .orElseThrow(() -> new IOException("Dataflow not found")); + .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); } @Override public DataStructure getStructure(DataStructureRef ref) throws IOException { return repo.getStructure(ref) - .orElseThrow(() -> new IOException("DataStructure not found")); + .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); } @Override - public DataCursor getData(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { + public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { return repo.getCursor(flowRef, query) - .orElseThrow(() -> new IOException("Data not found")); + .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); } @Override From abaecc3bd3a39ca1a49abe5692c5e8cd9f2d8f87 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 30 Nov 2017 16:39:06 +0100 Subject: [PATCH 58/68] Improved API. --- .../nbb/demetra/sdmx/web/SdmxWebProviderTest.java | 4 ++-- .../src/test/java/test/samples/FacadeResource.java | 8 ++++---- .../java/be/nbb/sdmx/facade/DataStructureRef.java | 4 ++++ .../main/java/be/nbb/sdmx/facade/DataflowRef.java | 4 ++++ .../java/be/nbb/sdmx/facade/DataflowRefTest.java | 6 ++++++ .../file/xml/DataStructureDecoderTest.java | 1 + .../be/nbb/sdmx/facade/tck/ConnectionAssert.java | 2 +- .../be/nbb/sdmx/facade/repo/SdmxRepository.java | 14 +++++++------- .../facade/xml/stream/XMLStreamStructure20.java | 1 + .../facade/xml/stream/XMLStreamStructure21.java | 1 + .../nbb/sdmx/facade/repo/SdmxRepositoryTest.java | 12 ++++++------ .../java/internal/web/sdmx21/Sdmx21RestClient.java | 4 ++-- .../src/test/java/test/client/RepoRestClient.java | 2 +- .../test/java/test/samples/ConnectorsResource.java | 8 ++++---- .../src/test/java/test/samples/FacadeResource.java | 8 ++++---- 15 files changed, 48 insertions(+), 31 deletions(-) diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java index 17ea4c4f..56b8af62 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/web/SdmxWebProviderTest.java @@ -127,8 +127,8 @@ private static SdmxRepository getCustomRepo() { return SdmxRepository.builder() .name("world") - .dataStructure(conjStruct) - .dataflow(conj) + .structure(conjStruct) + .flow(conj) .data(conj.getRef(), Series.builder().key(Key.of("BE", "IND")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 1)).build()) .data(conj.getRef(), Series.builder().key(Key.of("BE", "XXX")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 2)).build()) .data(conj.getRef(), Series.builder().key(Key.of("FR", "IND")).freq(Frequency.MONTHLY).obs(toSeries(TsFrequency.Monthly, 3)).build()) diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index afa8531a..c49a9422 100644 --- a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -57,8 +57,8 @@ public SdmxRepository nbb() throws IOException { List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); result = SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) + .structures(structs) + .flows(flows) .data(NBB_FLOW_REF, data) .name("NBB") .seriesKeysOnlySupported(false) @@ -79,8 +79,8 @@ public SdmxRepository ecb() throws IOException { List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); result = SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) + .structures(structs) + .flows(flows) .data(ECB_FLOW_REF, data) .name("ECB") .seriesKeysOnlySupported(true) diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java index b7130ab5..ec78578c 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java @@ -39,6 +39,10 @@ public final class DataStructureRef implements ResourceRef { @lombok.NonNull private String version; + public boolean equalsRef(@Nonnull DataStructure that) { + return equals(that.getRef()); + } + @Override public String toString() { return ResourceRefs.toString(this); diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java index e00ca7d4..5dfb9f35 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataflowRef.java @@ -48,6 +48,10 @@ public final class DataflowRef implements ResourceRef { @lombok.NonNull private String version; + public boolean containsRef(@Nonnull Dataflow that) { + return contains(that.getRef()); + } + public boolean contains(@Nonnull DataflowRef that) { return (this.agency.equals(ALL_AGENCIES) || this.agency.equals(that.agency)) && (this.id.equals(that.id)) diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java index 883932ec..e13c6975 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DataflowRefTest.java @@ -82,4 +82,10 @@ public void testContains() { assertThat(DataflowRef.of("world", "hello", "123").contains(DataflowRef.of("world", "hello", LATEST_VERSION))).isFalse(); assertThatNullPointerException().isThrownBy(() -> DataflowRef.of("world", "hello", "123").contains(null)); } + + @Test + @SuppressWarnings("null") + public void testContainsRef() { + assertThatNullPointerException().isThrownBy(() -> DataflowRef.of("world", "hello", "123").containsRef(null)); + } } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index 3ba7afb3..a68cace8 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static internal.file.xml.CustomDataStructureBuilder.dimension; import javax.xml.stream.XMLInputFactory; +import static internal.file.xml.CustomDataStructureBuilder.dimension; /** * diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 075a49ee..934259f1 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -51,7 +51,7 @@ public static void assertCompliance(SoftAssertions s, Callable s assertNonnull(s, conn, ref); DataCursorAssert.assertCompliance(s, () -> conn.getCursor(ref, ALL)); s.assertThat(conn.getStream(ref, ALL)).containsExactlyElementsOf(cursorToSeries(ref, ALL, conn)); - s.assertThat(conn.getFlows()).isNotEmpty().filteredOn(o -> ref.contains(o.getRef())).isNotEmpty(); + s.assertThat(conn.getFlows()).isNotEmpty().filteredOn(ref::containsRef).isNotEmpty(); s.assertThat(conn.getFlow(ref)).isNotNull(); s.assertThat(conn.getStructure(ref)).isNotNull(); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 24114f51..21d56683 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -51,11 +51,11 @@ public class SdmxRepository { @lombok.NonNull @lombok.Singular - List dataStructures; + List structures; @lombok.NonNull @lombok.Singular - Set dataflows; + Set flows; @lombok.NonNull Map> data; @@ -71,18 +71,18 @@ public SdmxConnection asConnection() { @Nonnull public Optional getStructure(@Nonnull DataStructureRef ref) { Objects.requireNonNull(ref); - return dataStructures + return structures .stream() - .filter(o -> o.getRef().equals(ref)) + .filter(ref::equalsRef) .findFirst(); } @Nonnull public Optional getFlow(@Nonnull DataflowRef ref) { Objects.requireNonNull(ref); - return dataflows + return flows .stream() - .filter(o -> ref.contains(o.getRef())) + .filter(ref::containsRef) .findFirst(); } @@ -156,7 +156,7 @@ private RepoConnection(SdmxRepository repo) { @Override public Set getFlows() throws IOException { checkState(); - return repo.getDataflows(); + return repo.getFlows(); } @Override diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index 673554c6..cd9db047 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -34,6 +34,7 @@ import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V20_URI; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 9da3dca3..97efc236 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -34,6 +34,7 @@ import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index c0c0dd43..096f2a3a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -81,13 +81,13 @@ public void testGetFlow() { } @Test - public void testGetDataStructures() { - assertThat(repo.getDataStructures()).containsExactly(struct); + public void testGetStructures() { + assertThat(repo.getStructures()).containsExactly(struct); } @Test - public void testGetDataflows() { - assertThat(repo.getDataflows()).containsExactly(flow); + public void testGetFlows() { + assertThat(repo.getFlows()).containsExactly(flow); } @Test @@ -118,8 +118,8 @@ public void testGetStructure() { private final SdmxRepository repo = SdmxRepository .builder() .name("test") - .dataStructure(struct) - .dataflow(flow) + .structure(struct) + .flow(flow) .data(goodFlowRef, series) .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java index 45a45b59..a36b1b61 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java @@ -66,7 +66,7 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { .asStreamParser(Stax.getInputFactory(), UTF_8) .parseWithIO(calling(url, XML)) .stream() - .filter(o -> ref.contains(o.getRef())) + .filter(ref::containsRef) .findFirst() .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); } @@ -78,7 +78,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { .asStreamParser(Stax.getInputFactory(), UTF_8) .parseWithIO(calling(url, STRUCTURE_21)) .stream() - .filter(o -> ref.equals(o.getRef())) + .filter(ref::equalsRef) .findFirst() .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java index 32dd3e8e..15b352bf 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/client/RepoRestClient.java @@ -41,7 +41,7 @@ public final class RepoRestClient implements RestClient { @Override public List getFlows() throws IOException { - return new ArrayList(repo.getDataflows()); + return new ArrayList(repo.getFlows()); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java index 1cd1584a..1312a311 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java @@ -63,8 +63,8 @@ public SdmxRepository nbb() throws IOException { DataflowRef ref = firstOf(flows); return SdmxRepository.builder() - .dataStructures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) - .dataflows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) + .structures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) + .flows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("NBB") .seriesKeysOnlySupported(false) @@ -82,8 +82,8 @@ public SdmxRepository ecb() throws IOException { DataflowRef ref = firstOf(flows); return SdmxRepository.builder() - .dataStructures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) - .dataflows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) + .structures(structs.stream().map(Util::toStructure).collect(Collectors.toList())) + .flows(flows.stream().map(Util::toFlow).collect(Collectors.toList())) .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) .name("ECB") .seriesKeysOnlySupported(true) diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java index afa8531a..c49a9422 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java @@ -57,8 +57,8 @@ public SdmxRepository nbb() throws IOException { List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); result = SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) + .structures(structs) + .flows(flows) .data(NBB_FLOW_REF, data) .name("NBB") .seriesKeysOnlySupported(false) @@ -79,8 +79,8 @@ public SdmxRepository ecb() throws IOException { List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); result = SdmxRepository.builder() - .dataStructures(structs) - .dataflows(flows) + .structures(structs) + .flows(flows) .data(ECB_FLOW_REF, data) .name("ECB") .seriesKeysOnlySupported(true) From 67e8a80dd8b02316ea04f1888a82ec1701c44dbf Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 30 Nov 2017 17:27:03 +0100 Subject: [PATCH 59/68] Improved IO API. --- .../java/test/samples/FacadeResource.java | 10 +- .../internal/file/SdmxDecoderResource.java | 3 +- .../internal/file/xml/StaxSdmxDecoder.java | 13 +- .../file/xml/DataStructureDecoderTest.java | 9 +- .../internal/file/xml/DataTypeProbeTest.java | 15 +- .../nbb/sdmx/facade/util/SeriesSupport.java | 6 +- .../src/main/java/be/nbb/util/IO.java | 471 ++++++++++++--- .../src/main/java/be/nbb/util/Stax.java | 67 ++- .../sdmx/facade/util/SeriesSupportTest.java | 8 +- .../xml/stream/XMLStreamFlow21Test.java | 2 +- .../xml/stream/XMLStreamStructure20Test.java | 6 +- .../xml/stream/XMLStreamStructure21Test.java | 6 +- .../src/test/java/be/nbb/util/IOTest.java | 539 +++++++++++++++--- .../src/test/java/be/nbb/util/StaxTest.java | 20 +- .../internal/web/sdmx21/RestExecutorImpl.java | 2 +- .../internal/web/sdmx21/Sdmx21RestClient.java | 8 +- .../src/test/java/test/DriverAssertions.java | 2 +- .../java/test/samples/FacadeResource.java | 10 +- 18 files changed, 973 insertions(+), 224 deletions(-) diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index c49a9422..3c204c21 100644 --- a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -95,7 +95,7 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference ECB = new AtomicReference<>(); private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.struct20(l).onReader(f).parseWithIO(xml::openReader); } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -106,21 +106,21 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(f, xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).parseWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.struct21(l).onReader(f).parseWithIO(xml::openReader); } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.flow21(l).onReader(f).parseWithIO(xml::openReader); } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(f, xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).parseWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java index fbfcfc07..77f62017 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java @@ -51,7 +51,8 @@ public SdmxDecoder.Info decode() throws IOException { @Override public DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { return getDataSupplier(entry.getType(), entry.getStructure()) - .parseFile(factoryWithoutNamespace, files.getData(), StandardCharsets.UTF_8); + .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) + .parseWithIO(files.getData()); } private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java index 506ce390..44ffce94 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java @@ -53,11 +53,16 @@ public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOExcep } private DataType probeDataType(File data) throws IOException { - return DataTypeProbe.of().parseFile(factory, data, StandardCharsets.UTF_8); + return DataTypeProbe.of() + .onFile(factory, StandardCharsets.UTF_8) + .parseWithIO(data); } private DataStructure parseStruct(DataType dataType, LanguagePriorityList langs, File structure) throws IOException { - return getStructParser(dataType, langs).parseFile(factory, structure, StandardCharsets.UTF_8).get(0); + return getStructParser(dataType, langs) + .onFile(factory, StandardCharsets.UTF_8) + .parseWithIO(structure) + .get(0); } private Stax.Parser> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { @@ -74,7 +79,9 @@ private Stax.Parser> getStructParser(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - return getStructDecoder(dataType).parseFile(factoryWithoutNamespace, data, StandardCharsets.UTF_8); + return getStructDecoder(dataType) + .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) + .parseWithIO(data); } private static Stax.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index a68cace8..4f57512a 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -23,7 +23,6 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.util.Stax; import static org.assertj.core.api.Assertions.assertThat; -import static internal.file.xml.CustomDataStructureBuilder.dimension; import javax.xml.stream.XMLInputFactory; import static internal.file.xml.CustomDataStructureBuilder.dimension; @@ -46,7 +45,7 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic20().parseReader(xif, SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic20().onReader(xif).parseWithIO(SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); } @Test @@ -63,7 +62,7 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact20().parseReader(xif, SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact20().onReader(xif).parseWithIO(SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); } @Test @@ -82,7 +81,7 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic21().parseReader(xif, SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic21().onReader(xif).parseWithIO(SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); } @Test @@ -101,7 +100,7 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact21().parseReader(xif, SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact21().onReader(xif).parseWithIO(SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); } private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java index b5f96721..25c11a97 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java @@ -17,10 +17,11 @@ package internal.file.xml; import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.util.IO; import be.nbb.util.Stax; import internal.file.SdmxDecoder; import java.io.IOException; -import javax.xml.stream.XMLInputFactory; +import java.io.Reader; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -32,27 +33,27 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { - assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_GENERIC20::openReader)) + assertThat(decoder.parseWithIO(SdmxSource.OTHER_GENERIC20::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC20); } @Test public void testDecodeCompact20() throws IOException { - assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_COMPACT20::openReader)) + assertThat(decoder.parseWithIO(SdmxSource.OTHER_COMPACT20::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT20); } @Test public void testDecodeGeneric21() throws IOException { - assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_GENERIC21::openReader)) + assertThat(decoder.parseWithIO(SdmxSource.OTHER_GENERIC21::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC21); } @Test public void testDecodeCompact21() throws IOException { - assertThat(DataTypeProbe.of().parseReader(xif, SdmxSource.OTHER_COMPACT21::openReader)) + assertThat(decoder.parseWithIO(SdmxSource.OTHER_COMPACT21::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT21); } - - private final XMLInputFactory xif = Stax.getInputFactory(); + + private final IO.Parser, SdmxDecoder.DataType> decoder = DataTypeProbe.of().onReader(Stax.getInputFactory()); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index 9cda698c..8358c786 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -79,14 +79,14 @@ public DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { } @Nonnull - public Stream asStream(@Nonnull IO.Supplier supplier) throws IOException { - return IO.stream(supplier, SeriesSupport::getDataStream); + public Stream asStream(@Nonnull IO.Supplier source) throws IOException { + return IO.Stream.open(source, SeriesSupport::getDataStream); } @SuppressWarnings("null") private Stream getDataStream(DataCursor cursor) { Series.Builder builder = Series.builder(); - return IO.streamNonnull(() -> cursor.nextSeries() ? getSeries(builder, cursor) : null); + return IO.Stream.generateUntilNull(() -> cursor.nextSeries() ? getSeries(builder, cursor) : null); } private Series getSeries(Series.Builder builder, DataCursor cursor) throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java index 35229c9a..d3a58c5a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java @@ -19,12 +19,16 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; -import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -36,11 +40,25 @@ @lombok.experimental.UtilityClass public class IO { + /** + * Represents a function without argument and result. + */ @FunctionalInterface public interface Runnable { + /** + * Run this function. + * + * @throws IOException + */ + @JdkWithIO void runWithIO() throws IOException; + @Nonnull + default Closeable asCloseable() { + return this::runWithIO; + } + @Nonnull default java.lang.Runnable asUnchecked() { return () -> { @@ -52,6 +70,22 @@ default java.lang.Runnable asUnchecked() { }; } + @Nonnull + static java.lang.Runnable unchecked(@Nonnull Runnable o) { + return o.asUnchecked(); + } + + @Nonnull + static Runnable checked(@Nonnull java.lang.Runnable o) { + return () -> { + try { + o.run(); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + }; + } + @Nonnull static Runnable throwing(@Nonnull java.util.function.Supplier ex) { Objects.requireNonNull(ex); @@ -67,9 +101,21 @@ static Runnable noOp() { } } + /** + * Represents a supplier of results. + * + * @param the type of results supplied by this supplier + */ @FunctionalInterface public interface Supplier { + /** + * Gets a result. + * + * @return a result + * @throws IOException + */ + @JdkWithIO T getWithIO() throws IOException; @Nonnull @@ -83,6 +129,23 @@ default java.util.function.Supplier asUnchecked() { }; } + @Nonnull + static java.util.function.Supplier unchecked(@Nonnull Supplier o) { + return o.asUnchecked(); + } + + @Nonnull + static Supplier checked(@Nonnull java.util.function.Supplier o) { + Objects.requireNonNull(o); + return () -> { + try { + return o.get(); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + }; + } + @Nonnull static Supplier throwing(@Nonnull java.util.function.Supplier ex) { Objects.requireNonNull(ex); @@ -98,11 +161,69 @@ static Supplier of(@Nullable T t) { } } + /** + * Represents a function that accepts one argument and produces a result. + * + * @param the type of the input to the function + * @param the type of the result of the function + */ @FunctionalInterface public interface Function { + /** + * Applies this function to the given argument. + * + * @param t the function argument + * @return the function result + * @throws java.io.IOException + */ + @JdkWithIO R applyWithIO(T t) throws IOException; + /** + * Returns a composed function that first applies the {@code before} + * function to its input, and then applies this function to the result. + * If evaluation of either function throws an exception, it is relayed + * to the caller of the composed function. + * + * @param the type of input to the {@code before} function, and to + * the composed function + * @param before the function to apply before this function is applied + * @return a composed function that first applies the {@code before} + * function and then applies this function + * @throws NullPointerException if before is null + * + * @see #andThen(FunctionWithIO) + */ + @JdkWithIO + @Nonnull + default Function compose(@Nonnull Function before) { + Objects.requireNonNull(before); + return (V v) -> applyWithIO(before.applyWithIO(v)); + } + + /** + * Returns a composed function that first applies this function to its + * input, and then applies the {@code after} function to the result. If + * evaluation of either function throws an exception, it is relayed to + * the caller of the composed function. + * + * @param the type of output of the {@code after} function, and of + * the composed function + * @param after the function to apply after this function is applied + * @return a composed function that first applies this function and then + * applies the {@code after} function + * @throws NullPointerException if after is null + * + * @see #compose(FunctionWithIO) + */ + @JdkWithIO + @Nonnull + default Function andThen(@Nonnull Function after) { + Objects.requireNonNull(after); + return (T t) -> after.applyWithIO(applyWithIO(t)); + } + @Nonnull default java.util.function.Function asUnchecked() { return (T t) -> { @@ -114,6 +235,23 @@ default java.util.function.Function asUnchecked() { }; } + @Nonnull + static java.util.function.Function unchecked(@Nonnull Function o) { + return o.asUnchecked(); + } + + @Nonnull + static Function checked(@Nonnull java.util.function.Function func) { + Objects.requireNonNull(func); + return o -> { + try { + return func.apply(o); + } catch (UncheckedIOException ex) { + throw ex.getCause(); + } + }; + } + @Nonnull static Function throwing(@Nonnull java.util.function.Supplier ex) { Objects.requireNonNull(ex); @@ -122,6 +260,18 @@ static Function throwing(@Nonnull java.util.function.Supplier the type of the input and output objects to the function + * @return a function that always returns its input argument + */ + @JdkWithIO + @Nonnull + static Function identity() { + return t -> t; + } + @Nonnull @SuppressWarnings("null") static Function of(@Nullable R r) { @@ -129,11 +279,86 @@ static Function of(@Nullable R r) { } } + /** + * Represents a predicate (boolean-valued function) of one argument. + * + * @param the type of the input to the predicate + */ @FunctionalInterface public interface Predicate { + /** + * Evaluates this predicate on the given argument. + * + * @param t the input argument + * @return {@code true} if the input argument matches the predicate, + * otherwise {@code false} + * @throws java.io.IOException + */ + @JdkWithIO boolean testWithIO(T t) throws IOException; + /** + * Returns a composed predicate that represents a short-circuiting + * logical AND of this predicate and another. When evaluating the + * composed predicate, if this predicate is {@code false}, then the + * {@code other} predicate is not evaluated. + * + *

+ * Any exceptions thrown during evaluation of either predicate are + * relayed to the caller; if evaluation of this predicate throws an + * exception, the {@code other} predicate will not be evaluated. + * + * @param other a predicate that will be logically-ANDed with this + * predicate + * @return a composed predicate that represents the short-circuiting + * logical AND of this predicate and the {@code other} predicate + * @throws NullPointerException if other is null + */ + @JdkWithIO + @Nonnull + default Predicate and(@Nonnull Predicate other) { + Objects.requireNonNull(other); + return (t) -> testWithIO(t) && other.testWithIO(t); + } + + /** + * Returns a predicate that represents the logical negation of this + * predicate. + * + * @return a predicate that represents the logical negation of this + * predicate + */ + @JdkWithIO + @Nonnull + default Predicate negate() { + return (t) -> !testWithIO(t); + } + + /** + * Returns a composed predicate that represents a short-circuiting + * logical OR of this predicate and another. When evaluating the + * composed predicate, if this predicate is {@code true}, then the + * {@code other} predicate is not evaluated. + * + *

+ * Any exceptions thrown during evaluation of either predicate are + * relayed to the caller; if evaluation of this predicate throws an + * exception, the {@code other} predicate will not be evaluated. + * + * @param other a predicate that will be logically-ORed with this + * predicate + * @return a composed predicate that represents the short-circuiting + * logical OR of this predicate and the {@code other} predicate + * @throws NullPointerException if other is null + */ + @JdkWithIO + @Nonnull + default Predicate or(@Nonnull Predicate other) { + Objects.requireNonNull(other); + return (t) -> testWithIO(t) || other.testWithIO(t); + } + @Nonnull default java.util.function.Predicate asUnchecked() { return (T t) -> { @@ -145,6 +370,23 @@ default java.util.function.Predicate asUnchecked() { }; } + @Nonnull + static java.util.function.Predicate unchecked(@Nonnull Predicate o) { + return o.asUnchecked(); + } + + @Nonnull + static Predicate checked(@Nonnull java.util.function.Predicate predicate) { + Objects.requireNonNull(predicate); + return o -> { + try { + return predicate.test(o); + } catch (UncheckedIOException ex) { + throw ex.getCause(); + } + }; + } + @Nonnull static Predicate throwing(@Nonnull java.util.function.Supplier ex) { Objects.requireNonNull(ex); @@ -153,17 +395,71 @@ static Predicate throwing(@Nonnull java.util.function.Supplier the type of arguments to the predicate + * @param targetRef the object reference with which to compare for + * equality, which may be {@code null} + * @return a predicate that tests if two arguments are equal according + * to {@link Objects#equals(Object, Object)} + */ + @JdkWithIO + @Nonnull + static Predicate isEqual(Object targetRef) { + return (null == targetRef) + ? Objects::isNull + : object -> targetRef.equals(object); + } + @Nonnull static Predicate of(boolean r) { return o -> r; } } + /** + * Represents an operation that accepts a single input argument and returns + * no result. Unlike most other functional interfaces, {@code Consumer} is + * expected to operate via side-effects. + * + * @param the type of the input to the operation + */ @FunctionalInterface public interface Consumer { + /** + * Performs this operation on the given argument. + * + * @param t the input argument + * @throws java.io.IOException + */ + @JdkWithIO void acceptWithIO(T t) throws IOException; + /** + * Returns a composed {@code Consumer} that performs, in sequence, this + * operation followed by the {@code after} operation. If performing + * either operation throws an exception, it is relayed to the caller of + * the composed operation. If performing this operation throws an + * exception, the {@code after} operation will not be performed. + * + * @param after the operation to perform after this operation + * @return a composed {@code Consumer} that performs in sequence this + * operation followed by the {@code after} operation + * @throws NullPointerException if {@code after} is null + */ + @JdkWithIO + @Nonnull + default Consumer andThen(@Nonnull Consumer after) { + Objects.requireNonNull(after); + return (T t) -> { + acceptWithIO(t); + after.acceptWithIO(t); + }; + } + @Nonnull default java.util.function.Consumer asUnchecked() { return (T t) -> { @@ -175,6 +471,22 @@ default java.util.function.Consumer asUnchecked() { }; } + @Nonnull + static java.util.function.Consumer unchecked(@Nonnull Consumer o) { + return o.asUnchecked(); + } + + @Nonnull + static Consumer checked(@Nonnull java.util.function.Consumer consumer) { + return o -> { + try { + consumer.accept(o); + } catch (UncheckedIOException ex) { + throw ex.getCause(); + } + }; + } + @Nonnull static Consumer throwing(@Nonnull java.util.function.Supplier ex) { Objects.requireNonNull(ex); @@ -193,82 +505,111 @@ static Consumer noOp() { @FunctionalInterface public interface Parser { - R parseWithIO(@Nonnull Supplier input) throws IOException; - } + R parseWithIO(@Nonnull T source) throws IOException; - @Nonnull - public Stream stream(@Nonnull Supplier supplier, @Nonnull Function> stream) throws IOException { - T resource = supplier.getWithIO(); - try { - return stream.applyWithIO(resource).onClose(asUnchecked(resource::close)); - } catch (Error | RuntimeException | IOException e) { - try { - resource.close(); - } catch (IOException ex) { - try { - e.addSuppressed(ex); - } catch (Throwable ignore) { + @Nonnull + static Parser valueOf( + @Nonnull Function opener, + @Nonnull Function reader, + @Nonnull Consumer closer) { + Objects.requireNonNull(opener); + Objects.requireNonNull(reader); + Objects.requireNonNull(closer); + return source -> { + R resource = opener.applyWithIO(source); + try (Closeable c = () -> closer.acceptWithIO(resource)) { + return reader.applyWithIO(resource); } - } - throw e; + }; } - } - - @Nonnull - public Stream streamNonnull(@Nonnull Supplier nextSupplier) { - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(asIterator(nextSupplier), Spliterator.ORDERED | Spliterator.NONNULL), false); - } - @Nonnull - private Iterator asIterator(@Nonnull Supplier nextSupplier) { - Objects.requireNonNull(nextSupplier); - return new Iterator() { - T nextElement = null; - - @Override - public boolean hasNext() { - if (nextElement != null) { - return true; - } else { + @Nonnull + static Parser flowOf( + @Nonnull Function opener, + @Nonnull Function reader, + @Nonnull Consumer closer) { + Objects.requireNonNull(opener); + Objects.requireNonNull(reader); + Objects.requireNonNull(closer); + return source -> { + R resource = opener.applyWithIO(source); + try { + return reader.applyWithIO(resource); + } catch (Error | RuntimeException | IOException e) { try { - nextElement = nextSupplier.getWithIO(); - return (nextElement != null); - } catch (IOException e) { - throw new UncheckedIOException(e); + closer.acceptWithIO(resource); + } catch (IOException ex) { + try { + e.addSuppressed(ex); + } catch (Throwable ignore) { + } } + throw e; } - } - - @Override - public T next() { - if (nextElement != null || hasNext()) { - T line = nextElement; - nextElement = null; - return line; - } else { - throw new NoSuchElementException(); - } - } - }; + }; + } } - @Nonnull - public java.lang.Runnable asUnchecked(@Nonnull Runnable o) { - return o.asUnchecked(); - } + @lombok.experimental.UtilityClass + public static final class Stream { - @Nonnull - public java.util.function.Function asUnchecked(@Nonnull Function o) { - return o.asUnchecked(); - } + @Nonnull + public java.util.stream.Stream open(@Nonnull Supplier source, @Nonnull Function> streamer) throws IOException { + return asParser(streamer).parseWithIO(source); + } - @Nonnull - public java.util.function.Predicate asUnchecked(@Nonnull Predicate o) { - return o.asUnchecked(); + @Nonnull + public java.util.stream.Stream generateUntilNull(@Nonnull Supplier generator) { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(asIterator(generator), Spliterator.ORDERED | Spliterator.NONNULL), false); + } + + private Parser, java.util.stream.Stream> asParser(Function> streamer) { + return Parser.flowOf( + source -> source.getWithIO(), + resource -> streamer.applyWithIO(resource).onClose(Runnable.unchecked(resource::close)), + Closeable::close + ); + } + + @Nonnull + private Iterator asIterator(@Nonnull Supplier nextSupplier) { + Objects.requireNonNull(nextSupplier); + return new Iterator() { + T nextElement = null; + + @Override + public boolean hasNext() { + if (nextElement != null) { + return true; + } else { + try { + nextElement = nextSupplier.getWithIO(); + return (nextElement != null); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Override + public T next() { + if (nextElement != null || hasNext()) { + T line = nextElement; + nextElement = null; + return line; + } else { + throw new NoSuchElementException(); + } + } + }; + } } - @Nonnull - public java.util.function.Consumer asUnchecked(@Nonnull Consumer o) { - return o.asUnchecked(); + @Target({ElementType.METHOD}) + @Retention(RetentionPolicy.SOURCE) + @Documented + private @interface JdkWithIO { + + String value() default ""; } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java index c27d199d..2b09cd54 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java @@ -63,45 +63,29 @@ public interface Parser { T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; @Nonnull - default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull File file, @Nonnull Charset cs) throws IOException { - return parseStream(xf, () -> new FileInputStream(file), cs); + default IO.Parser onFile(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + return source -> parseStream(this, xf, () -> new FileInputStream(source), cs); } @Nonnull - default T parseFile(@Nonnull XMLInputFactory xf, @Nonnull Path path, @Nonnull Charset cs) throws IOException { - try { - return parseStream(xf, () -> new FileInputStream(path.toFile()), cs); - } catch (UnsupportedOperationException ex) { - return parseReader(xf, () -> Files.newBufferedReader(path, cs)); - } - } - - @Nonnull - default T parseStream(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source, @Nonnull Charset cs) throws IOException { - InputStream stream = source.getWithIO(); - return parse(() -> xf.createXMLStreamReader(stream, cs.name()), stream); - } - - @Nonnull - default T parseReader(@Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source) throws IOException { - Reader reader = source.getWithIO(); - return parse(() -> xf.createXMLStreamReader(reader), reader); + default IO.Parser onPath(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + return source -> { + try { + return parseStream(this, xf, () -> new FileInputStream(source.toFile()), cs); + } catch (UnsupportedOperationException ex) { + return parseReader(this, xf, () -> Files.newBufferedReader(source, cs)); + } + }; } @Nonnull - default T parse(@Nonnull Supplier supplier, @Nonnull Closeable onClose) throws IOException { - try { - XMLStreamReader xml = supplier.getWithStream(); - return parse(xml, () -> closeBoth(xml, onClose)); - } catch (XMLStreamException ex) { - ensureClosed(ex, onClose); - throw new XMLStreamIOException("Failed to create XMLStreamReader", ex); - } + default IO.Parser, T> onInputStream(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + return source -> parseStream(this, xf, source, cs); } @Nonnull - default IO.Parser asStreamParser(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { - return o -> parseStream(xf, o, cs); + default IO.Parser, T> onReader(@Nonnull XMLInputFactory xf) { + return source -> parseReader(this, xf, source); } @Nonnull @@ -116,6 +100,29 @@ static Parser of(@Nonnull Function func) { } } + @Nonnull + private T parse(Parser parser, @Nonnull Supplier supplier, @Nonnull Closeable onClose) throws IOException { + try { + XMLStreamReader xml = supplier.getWithStream(); + return parser.parse(xml, () -> closeBoth(xml, onClose)); + } catch (XMLStreamException ex) { + ensureClosed(ex, onClose); + throw new XMLStreamIOException("Failed to create XMLStreamReader", ex); + } + } + + @Nonnull + T parseStream(Parser parser, @Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source, @Nonnull Charset cs) throws IOException { + InputStream stream = source.getWithIO(); + return parse(parser, () -> xf.createXMLStreamReader(stream, cs.name()), stream); + } + + @Nonnull + T parseReader(Parser parser, @Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source) throws IOException { + Reader reader = source.getWithIO(); + return parse(parser, () -> xf.createXMLStreamReader(reader), reader); + } + public static final class XMLStreamIOException extends IOException { public XMLStreamIOException(XMLStreamException ex) { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index bb15f467..b1a12901 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -76,8 +76,8 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).parseWithIO(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } } @@ -99,8 +99,8 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(ANY).parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(xif, SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).parseWithIO(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(120); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java index 9f806cce..e401bbb3 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java @@ -38,7 +38,7 @@ public class XMLStreamFlow21Test { public void test() throws IOException { Stax.Parser> p = Stax.Parser.of(new XMLStreamFlow21(LanguagePriorityList.ANY)::parse); - assertThat(p.parseReader(xif, SdmxSource.ECB_DATAFLOWS::openReader)) + assertThat(p.onReader(xif).parseWithIO(SdmxSource.ECB_DATAFLOWS::openReader)) .containsExactly( Dataflow.of(DataflowRef.of("ECB", "AME", "1.0"), DataStructureRef.of("ECB", "ECB_AME1", "1.0"), "AMECO"), Dataflow.of(DataflowRef.of("ECB", "BKN", "1.0"), DataStructureRef.of("ECB", "ECB_BKN1", "1.0"), "Banknotes statistics"), diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index eea0a815..f68ae4a1 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -39,7 +39,7 @@ public class XMLStreamStructure20Test { public void test() throws Exception { Stax.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - assertThat(p1.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p1.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -53,7 +53,7 @@ public void test() throws Exception { Stax.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - assertThat(p2.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p2.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -66,7 +66,7 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> p1.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> p1.onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index dca06284..c7952370 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -38,7 +38,7 @@ public class XMLStreamStructure21Test { public void test() throws Exception { Stax.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - assertThat(parser.parseReader(xif, SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(parser.onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); @@ -51,10 +51,10 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> parser.parseReader(xif, SdmxSource.NBB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> parser.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } - + private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java index 5b0727ca..7fcb0d9e 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java @@ -17,12 +17,14 @@ package be.nbb.util; import java.io.Closeable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -33,174 +35,565 @@ */ public class IOTest { + // + private static IO.Function> streamerOf(R... values) { + return o -> Stream.of(values); + } + @Test @SuppressWarnings("null") - public void testStream() { - assertThatNullPointerException().isThrownBy(() -> IO.stream(null, emptyStream())); - assertThatNullPointerException().isThrownBy(() -> IO.stream(() -> null, null)); + public void testStreamOpen() throws IOException { + assertThatNullPointerException().isThrownBy(() -> IO.Stream.open(null, streamerOf())); + assertThatNullPointerException().isThrownBy(() -> IO.Stream.open(() -> null, null)); - IO.Supplier factoryError = IO.Supplier.throwing(FactoryError::new); - IO.Supplier closeError = () -> IO.Runnable.throwing(CloseError::new)::runWithIO; - IO.Function> streamError = IO.Function.throwing(StreamError::new); + IO.Supplier ofOpenError = IO.Supplier.throwing(OpenError::new); + IO.Supplier ofCloseError = () -> IO.Runnable.throwing(CloseError::new)::runWithIO; + IO.Function> toReadError = IO.Function.throwing(ReadError::new); - assertThatThrownBy(() -> IO.stream(factoryError, emptyStream()).close()) - .isInstanceOf(FactoryError.class) + assertThatThrownBy(() -> IO.Stream.open(ofOpenError, streamerOf()).close()) + .isInstanceOf(OpenError.class) .hasNoSuppressedExceptions(); - assertThatThrownBy(() -> IO.stream(factoryError, streamError).close()) - .isInstanceOf(FactoryError.class) + assertThatThrownBy(() -> IO.Stream.open(ofOpenError, toReadError).close()) + .isInstanceOf(OpenError.class) .hasNoSuppressedExceptions(); - assertThatThrownBy(() -> IO.stream(closeError, emptyStream()).close()) + assertThatThrownBy(() -> IO.Stream.open(ofCloseError, streamerOf()).close()) .isInstanceOf(UncheckedIOException.class) .hasRootCauseInstanceOf(CloseError.class) .hasNoSuppressedExceptions(); - assertThatThrownBy(() -> IO.stream(closeError, streamError).close()) - .isInstanceOf(StreamError.class) + assertThatThrownBy(() -> IO.Stream.open(ofCloseError, toReadError).close()) + .isInstanceOf(ReadError.class) .hasSuppressedException(new CloseError()); - assertThat(new AtomicBoolean(false)).satisfies(c -> { - assertThatThrownBy(() -> IO.stream(closeable(c), streamError).close()) - .isInstanceOf(StreamError.class) + assertThat(new AtomicInteger(0)).satisfies(c -> { + assertThatThrownBy(() -> IO.Stream.open(() -> c::incrementAndGet, toReadError).close()) + .isInstanceOf(ReadError.class) .hasNoSuppressedExceptions(); - assertThat(c).isTrue(); + assertThat(c).hasValue(1); }); - assertThat(new AtomicBoolean(false)).satisfies(c -> { - assertThatCode(() -> IO.stream(closeable(c), emptyStream()).close()).doesNotThrowAnyException(); - assertThat(c).isTrue(); + assertThat(new AtomicInteger(0)).satisfies(c -> { + assertThatCode(() -> IO.Stream.open(() -> c::incrementAndGet, streamerOf()).close()).doesNotThrowAnyException(); + assertThat(c).hasValue(1); }); - assertThat(new AtomicBoolean(false)).satisfies(c -> { - assertThatCode(() -> IO.stream(closeable(c), emptyStream())).doesNotThrowAnyException(); - assertThat(c).isFalse(); + assertThat(new AtomicInteger(0)).satisfies(c -> { + assertThatCode(() -> IO.Stream.open(() -> c::incrementAndGet, streamerOf())).doesNotThrowAnyException(); + assertThat(c).hasValue(0); }); + + assertThat(IO.Stream.open(IO.Runnable.noOp()::asCloseable, streamerOf())).isEmpty(); + assertThat(IO.Stream.open(IO.Runnable.noOp()::asCloseable, streamerOf("a", "b", "c"))).hasSize(3); } @Test @SuppressWarnings("null") - public void testStreamNonnull() { - assertThatNullPointerException().isThrownBy(() -> IO.streamNonnull(null)); + public void testStreamGenerateUntilNull() { + assertThatNullPointerException().isThrownBy(() -> IO.Stream.generateUntilNull(null)); - assertThat(IO.streamNonnull(() -> null)).isEmpty(); + assertThat(IO.Stream.generateUntilNull(() -> null)).isEmpty(); Iterator iter = Arrays.asList("A", "B").iterator(); - assertThat(IO.streamNonnull(() -> iter.hasNext() ? iter.next() : null)).containsExactly("A", "B"); + assertThat(IO.Stream.generateUntilNull(() -> iter.hasNext() ? iter.next() : null)).containsExactly("A", "B"); + + assertThatThrownBy(() -> IO.Stream.generateUntilNull(ofError1).count()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + // + + // + private final IO.Function toUpperCase = String::toUpperCase; + private final IO.Function toString = Object::toString; + private final IO.Function toError1 = IO.Function.throwing(Error1::new); + private final IO.Function toError2 = IO.Function.throwing(Error2::new); + + @Test + @SuppressWarnings("null") + public void testFunctionCompose() throws IOException { + assertThatNullPointerException().isThrownBy(() -> toUpperCase.compose(null)); + + assertThat(toUpperCase.compose(toString).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString().toUpperCase()); + assertThatThrownBy(() -> toError1.compose(toString).applyWithIO(Byte.class)).isInstanceOf(Error1.class); + assertThatThrownBy(() -> toString.compose(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); + assertThatThrownBy(() -> toError1.compose(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); + } + + @Test + @SuppressWarnings("null") + public void testFunctionAndThen() throws IOException { + assertThatNullPointerException().isThrownBy(() -> toUpperCase.andThen(null)); + + assertThat(toString.andThen(toUpperCase).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString().toUpperCase()); + assertThatThrownBy(() -> toError1.andThen(toString).applyWithIO(Byte.class)).isInstanceOf(Error1.class); + assertThatThrownBy(() -> toString.andThen(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); + assertThatThrownBy(() -> toError1.andThen(toError2).applyWithIO(Byte.class)).isInstanceOf(Error1.class); + } + + @Test + public void testFunctionAsUnchecked() { + assertThat(toUpperCase.asUnchecked().apply("hello")).isEqualTo("HELLO"); + assertThatThrownBy(() -> toError1.asUnchecked().apply(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testFunctionUnchecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Function.unchecked(null)); - assertThatThrownBy(() -> IO.streamNonnull(IO.Supplier.throwing(FileNotFoundException::new)).count()) + assertThat(IO.Function.unchecked(toUpperCase).apply("hello")).isEqualTo("HELLO"); + assertThatThrownBy(() -> IO.Function.unchecked(toError1).apply(null)) .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + .hasCauseExactlyInstanceOf(Error1.class); } @Test @SuppressWarnings("null") - public void testShortcuts() { - assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Runnable) null)); - assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Function) null)); - assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Predicate) null)); - assertThatNullPointerException().isThrownBy(() -> IO.asUnchecked((IO.Consumer) null)); + public void testFunctionChecked() throws IOException { + assertThatNullPointerException().isThrownBy(() -> IO.Function.checked(null)); + + assertThat(IO.Function.checked(Object::toString).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString()); + assertThatThrownBy(() -> IO.Function.checked(IO.Function.unchecked(toError1)).applyWithIO(null)) + .isInstanceOf(Error1.class); } @Test @SuppressWarnings("null") public void testFunctionThrowing() { assertThatNullPointerException().isThrownBy(() -> IO.Function.throwing(null)); - assertThatThrownBy(() -> IO.Function.throwing(FileNotFoundException::new).applyWithIO(null)) - .isInstanceOf(FileNotFoundException.class); + + assertThatThrownBy(() -> IO.Function.throwing(Error1::new).applyWithIO(null)).isInstanceOf(Error1.class); } @Test - public void testFunctionAsUnchecked() { - assertThatCode(() -> IO.Function.of("").asUnchecked().apply("")).doesNotThrowAnyException(); + public void testFunctionIdentity() throws IOException { + assertThat(IO.Function.identity().applyWithIO(null)).isNull(); + assertThat(IO.Function.identity().applyWithIO(Byte.class)).isEqualTo(Byte.class); + } + + @Test + public void testFunctionOf() throws IOException { + assertThat(IO.Function.of(null).applyWithIO(null)).isNull(); + assertThat(IO.Function.of("").applyWithIO(Byte.class)).isEqualTo(""); + } + // + + // + private final IO.Runnable onError1 = IO.Runnable.throwing(Error1::new); + + @Test + public void testRunnableAsCloseable() { + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatCode(() -> ((IO.Runnable) o::incrementAndGet).asCloseable().close()).doesNotThrowAnyException(); + assertThat(o).hasValue(1); + }); + + assertThatThrownBy(() -> onError1.asCloseable().close()).isInstanceOf(Error1.class); + } + + @Test + public void testRunnableAsUnchecked() { + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatCode(() -> ((IO.Runnable) o::incrementAndGet).asUnchecked().run()).doesNotThrowAnyException(); + assertThat(o).hasValue(1); + }); - assertThatThrownBy(() -> IO.Function.throwing(FileNotFoundException::new).asUnchecked().apply(null)) + assertThatThrownBy(() -> onError1.asUnchecked().run()) .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testRunnableUnchecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Runnable.unchecked(null)); + + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatCode(() -> IO.Runnable.unchecked(o::incrementAndGet).run()).doesNotThrowAnyException(); + assertThat(o).hasValue(1); + }); + + assertThatThrownBy(() -> IO.Runnable.unchecked(onError1).run()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testRunnableChecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Runnable.unchecked(null)); + + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatCode(() -> IO.Runnable.checked(IO.Runnable.unchecked(o::incrementAndGet)).runWithIO()).doesNotThrowAnyException(); + assertThat(o).hasValue(1); + }); + + assertThatThrownBy(() -> IO.Runnable.checked(IO.Runnable.unchecked(onError1)).runWithIO()).isInstanceOf(Error1.class); } @Test @SuppressWarnings("null") public void testRunnableThrowing() { assertThatNullPointerException().isThrownBy(() -> IO.Runnable.throwing(null)); - assertThatThrownBy(() -> IO.Runnable.throwing(FileNotFoundException::new).runWithIO()) - .isInstanceOf(FileNotFoundException.class); + + assertThatThrownBy(() -> onError1.runWithIO()).isInstanceOf(Error1.class); } @Test - public void testRunnableAsUnchecked() { - assertThatCode(() -> IO.Runnable.noOp().asUnchecked().run()).doesNotThrowAnyException(); + public void testRunnableNoOp() { + assertThatCode(() -> IO.Runnable.noOp().runWithIO()).doesNotThrowAnyException(); + } + // + + // + private final IO.Supplier ofError1 = IO.Supplier.throwing(Error1::new); + private final IO.Supplier ofIncrement = new AtomicInteger()::incrementAndGet; - assertThatThrownBy(() -> IO.Runnable.throwing(FileNotFoundException::new).asUnchecked().run()) + @Test + public void testSupplierAsUnchecked() throws IOException { + assertThat(ofIncrement.asUnchecked().get()).isEqualTo(ofIncrement.getWithIO() - 1); + assertThatThrownBy(() -> ofError1.asUnchecked().get()) .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testSupplierUnchecked() throws IOException { + assertThatNullPointerException().isThrownBy(() -> IO.Supplier.unchecked(null)); + + assertThat(IO.Supplier.unchecked(ofIncrement).get()).isEqualTo(ofIncrement.getWithIO() - 1); + assertThatThrownBy(() -> IO.Supplier.unchecked(ofError1).get()) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testSupplierChecked() throws IOException { + assertThatNullPointerException().isThrownBy(() -> IO.Supplier.checked(null)); + + assertThat(IO.Supplier.checked(IO.Supplier.unchecked(ofIncrement)).getWithIO()).isEqualTo(ofIncrement.getWithIO() - 1); + assertThatThrownBy(() -> IO.Supplier.checked(IO.Supplier.unchecked(ofError1)).getWithIO()).isInstanceOf(Error1.class); } @Test @SuppressWarnings("null") public void testSupplierThrowing() { assertThatNullPointerException().isThrownBy(() -> IO.Supplier.throwing(null)); - assertThatThrownBy(() -> IO.Supplier.throwing(FileNotFoundException::new).getWithIO()) - .isInstanceOf(FileNotFoundException.class); + assertThatThrownBy(() -> ofError1.getWithIO()).isInstanceOf(Error1.class); + } + + @Test + public void testSupplierOf() throws IOException { + assertThat(IO.Supplier.of(null).getWithIO()).isNull(); + assertThat(IO.Supplier.of("").getWithIO()).isEqualTo(""); + } + // + + // + private final IO.Predicate isNonNull = Objects::nonNull; + private final IO.Predicate isNull = Objects::isNull; + private final IO.Predicate isNotEmptyString = o -> ((String) o).length() > 0; + private final IO.Predicate isError1 = IO.Predicate.throwing(Error1::new); + private final IO.Predicate isError2 = IO.Predicate.throwing(Error2::new); + + @Test + @SuppressWarnings("null") + public void testPredicateAnd() throws IOException { + assertThatNullPointerException().isThrownBy(() -> isNonNull.and(null)); + + assertThat(isNonNull.and(isNotEmptyString).testWithIO("a")).isTrue(); + assertThat(isNonNull.and(isNotEmptyString).testWithIO("")).isFalse(); + assertThat(isNonNull.and(isNotEmptyString).testWithIO(null)).isFalse(); + assertThatThrownBy(() -> isNonNull.and(isError1).testWithIO("")).isInstanceOf(Error1.class); + assertThatThrownBy(() -> isError1.and(isNonNull).testWithIO("")).isInstanceOf(Error1.class); + assertThatThrownBy(() -> isError1.and(isError2).testWithIO("")).isInstanceOf(Error1.class); + } + + @Test + public void testPredicateNegate() throws IOException { + assertThat(isNonNull.negate().testWithIO(null)).isTrue(); + assertThat(isNonNull.negate().testWithIO("")).isFalse(); + assertThatThrownBy(() -> isError1.negate().testWithIO("")).isInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testPredicateOr() throws IOException { + assertThatNullPointerException().isThrownBy(() -> isNonNull.or(null)); + + assertThat(isNull.or(isNotEmptyString).testWithIO("a")).isTrue(); + assertThat(isNull.or(isNotEmptyString).testWithIO("")).isFalse(); + assertThat(isNull.or(isNotEmptyString).testWithIO(null)).isTrue(); + assertThat(isNull.or(isError1).testWithIO(null)).isTrue(); + assertThatThrownBy(() -> isNull.or(isError1).testWithIO("")).isInstanceOf(Error1.class); + assertThatThrownBy(() -> isError1.or(isNull).testWithIO("")).isInstanceOf(Error1.class); + assertThatThrownBy(() -> isError1.or(isError2).testWithIO("")).isInstanceOf(Error1.class); } @Test - public void testSupplierAsUnchecked() { - assertThatCode(() -> IO.Supplier.of("").asUnchecked().get()).doesNotThrowAnyException(); + public void testPredicateAsUnchecked() { + assertThat(IO.Predicate.of(true).asUnchecked().test("")).isTrue(); + assertThatThrownBy(() -> isError1.asUnchecked().test(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testPredicateUnchecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Predicate.unchecked(null)); - assertThatThrownBy(() -> IO.Supplier.throwing(FileNotFoundException::new).asUnchecked().get()) + assertThat(IO.Predicate.unchecked(IO.Predicate.of(true)).test("")).isTrue(); + assertThatThrownBy(() -> IO.Predicate.unchecked(isError1).test(null)) .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testPredicateChecked() throws IOException { + assertThatNullPointerException().isThrownBy(() -> IO.Predicate.checked(null)); + + assertThat(IO.Predicate.checked(o -> true).testWithIO("")).isTrue(); + assertThatThrownBy(() -> IO.Predicate.checked(IO.Predicate.unchecked(isError1)).testWithIO(null)).isInstanceOf(Error1.class); } @Test @SuppressWarnings("null") public void testPredicateThrowing() { assertThatNullPointerException().isThrownBy(() -> IO.Predicate.throwing(null)); - assertThatThrownBy(() -> IO.Predicate.throwing(FileNotFoundException::new).testWithIO(null)) - .isInstanceOf(FileNotFoundException.class); + + assertThatThrownBy(() -> isError1.testWithIO(null)).isInstanceOf(Error1.class); } @Test - public void testPredicateAsUnchecked() { - assertThatCode(() -> IO.Predicate.of(true).asUnchecked().test("")).doesNotThrowAnyException(); + @SuppressWarnings("null") + public void testPredicateIsEqual() throws IOException { + assertThat(IO.Predicate.isEqual(null).testWithIO(null)).isTrue(); + assertThat(IO.Predicate.isEqual(null).testWithIO("")).isFalse(); + assertThat(IO.Predicate.isEqual("").testWithIO("")).isTrue(); + assertThat(IO.Predicate.isEqual("").testWithIO(null)).isFalse(); + } + + @Test + public void testPredicateOf() throws IOException { + assertThat(IO.Predicate.of(true).testWithIO(null)).isTrue(); + assertThat(IO.Predicate.of(false).testWithIO("")).isFalse(); + } + // + + // + private final IO.Consumer withError1 = IO.Consumer.throwing(Error1::new); + private final IO.Consumer withError2 = IO.Consumer.throwing(Error2::new); + private final IO.Consumer withIncrement = AtomicInteger::incrementAndGet; + + @Test + @SuppressWarnings("null") + public void testConsumerAndThen() { + assertThatNullPointerException().isThrownBy(() -> withIncrement.andThen(null)); - assertThatThrownBy(() -> IO.Predicate.throwing(FileNotFoundException::new).asUnchecked().test(null)) + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatCode(() -> withIncrement.andThen(withIncrement).acceptWithIO(o)).doesNotThrowAnyException(); + assertThat(o.get()).isEqualTo(2); + }); + + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatThrownBy(() -> withError1.andThen(withIncrement).acceptWithIO(o)).isInstanceOf(Error1.class); + assertThat(o.get()).isEqualTo(0); + }); + + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatThrownBy(() -> withIncrement.andThen(withError1).acceptWithIO(o)).isInstanceOf(Error1.class); + assertThat(o.get()).isEqualTo(1); + }); + + assertThat(new AtomicInteger(0)).satisfies(o -> { + assertThatThrownBy(() -> withError1.andThen(withError2).acceptWithIO(o)).isInstanceOf(Error1.class); + assertThat(o.get()).isEqualTo(0); + }); + } + + @Test + public void testConsumerAsUnchecked() { + assertThatCode(() -> IO.Consumer.noOp().asUnchecked().accept(null)).doesNotThrowAnyException(); + assertThatThrownBy(() -> withError1.asUnchecked().accept(null)) .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testConsumerUnchecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Consumer.unchecked(null)); + + assertThatCode(() -> IO.Consumer.unchecked(IO.Consumer.noOp()).accept(null)).doesNotThrowAnyException(); + assertThatThrownBy(() -> IO.Consumer.unchecked(withError1).accept(null)) + .isInstanceOf(UncheckedIOException.class) + .hasCauseExactlyInstanceOf(Error1.class); + } + + @Test + @SuppressWarnings("null") + public void testConsumerChecked() { + assertThatNullPointerException().isThrownBy(() -> IO.Consumer.unchecked(null)); + + assertThatCode(() -> IO.Consumer.checked(IO.Consumer.unchecked(IO.Consumer.noOp())).acceptWithIO(null)).doesNotThrowAnyException(); + assertThatThrownBy(() -> IO.Consumer.checked(IO.Consumer.unchecked(withError1)).acceptWithIO(null)).isInstanceOf(Error1.class); } @Test @SuppressWarnings("null") public void testConsumerThrowing() { assertThatNullPointerException().isThrownBy(() -> IO.Consumer.throwing(null)); - assertThatThrownBy(() -> IO.Consumer.throwing(FileNotFoundException::new).acceptWithIO(null)) - .isInstanceOf(FileNotFoundException.class); + + assertThatThrownBy(() -> IO.Consumer.throwing(Error1::new).acceptWithIO(null)).isInstanceOf(Error1.class); } @Test - public void testConsumerAsUnchecked() { - assertThatCode(() -> IO.Consumer.noOp().asUnchecked().accept("")).doesNotThrowAnyException(); + public void testConsumerNoOp() { + assertThatCode(() -> IO.Consumer.noOp().acceptWithIO(null)).doesNotThrowAnyException(); + } + // - assertThatThrownBy(() -> IO.Consumer.throwing(FileNotFoundException::new).asUnchecked().accept(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(FileNotFoundException.class); + @lombok.AllArgsConstructor + private static final class X implements Closeable { + + private final List events; + + static X open(List stack) throws IOException { + X result = new X(stack); + result.events.add("open"); + return result; + } + + public Closeable read() throws IOException { + events.add("read"); + return this; + } + + @Override + public void close() throws IOException { + events.add("close"); + } } - private static IO.Function> emptyStream() { - return IO.Function.of(Stream.empty()); + @Test + public void testParserValueOf() throws IOException { + IO.Function openError = IO.Function.throwing(OpenError::new); + IO.Function readError = IO.Function.throwing(ReadError::new); + IO.Consumer closeError = IO.Consumer.throwing(CloseError::new); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Object> p = IO.Parser.valueOf(X::open, X::read, X::close); + assertThatCode(() -> p.parseWithIO((List) c)).doesNotThrowAnyException(); + assertThat(c).containsExactly("open", "read", "close"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Object> p = IO.Parser.valueOf(openError, X::read, X::close); + assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(OpenError.class); + assertThat(c).isEmpty(); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Object> p = IO.Parser.valueOf(X::open, readError, X::close); + assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(ReadError.class); + assertThat(c).containsExactly("open", "close"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Object> p = IO.Parser.valueOf(X::open, X::read, closeError); + assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(CloseError.class); + assertThat(c).containsExactly("open", "read"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Object> p = IO.Parser.valueOf(X::open, readError, closeError); + assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(ReadError.class).hasSuppressedException(new CloseError()); + assertThat(c).containsExactly("open"); + }); + } + + @Test + public void testParserFlowOf() throws IOException { + IO.Function openError = IO.Function.throwing(OpenError::new); + IO.Function readError = IO.Function.throwing(ReadError::new); + IO.Consumer closeError = IO.Consumer.throwing(CloseError::new); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.flowOf(X::open, X::read, X::close); + assertThatCode(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + } + }).doesNotThrowAnyException(); + assertThat(c).containsExactly("open", "read", "close"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.flowOf(X::open, X::read, X::close); + assertThatThrownBy(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + throw new Error1(); + } + }).isInstanceOf(Error1.class); + assertThat(c).containsExactly("open", "read", "close"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.valueOf(openError, X::read, X::close); + assertThatThrownBy(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + } + }).isInstanceOf(OpenError.class); + assertThat(c).isEmpty(); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, readError, X::close); + assertThatThrownBy(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + } + }).isInstanceOf(ReadError.class); + assertThat(c).containsExactly("open", "close"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, X::read, closeError); + assertThatThrownBy(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + } + }).isInstanceOf(CloseError.class); + assertThat(c).containsExactly("open", "read"); + }); + + assertThat(new ArrayList()).satisfies(c -> { + IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, readError, closeError); + assertThatThrownBy(() -> { + try (AutoCloseable auto = p.parseWithIO((List) c)) { + } + }).isInstanceOf(ReadError.class).hasSuppressedException(new CloseError()); + assertThat(c).containsExactly("open"); + }); } - private static IO.Supplier closeable(AtomicBoolean o) { - return IO.Supplier.of(() -> o.set(true)); + private static final class OpenError extends IOException { } - private static final class FactoryError extends IOException { + private static final class ReadError extends IOException { } private static final class CloseError extends IOException { } - private static final class StreamError extends IOException { + private static final class Error1 extends IOException { + } + + private static final class Error2 extends IOException { } } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java index 0427d526..94a4d367 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java @@ -39,22 +39,22 @@ public void testValidParseStream() throws IOException { return ""; }; - assertThatNullPointerException().isThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)); + assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, null, EmptyStream::new, UTF_8)); - assertThatNullPointerException().isThrownBy(() -> p.parseStream(xif, null, UTF_8)); + assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, xif, null, UTF_8)); - assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, OpenErrorStream::new, UTF_8)) .isInstanceOf(OpenError.class) .hasNoSuppressedExceptions() .hasNoCause(); - assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, ReadErrorStream::new, UTF_8)) .isInstanceOf(Stax.XMLStreamIOException.class) .hasNoSuppressedExceptions() .hasCauseInstanceOf(XMLStreamException.class) .hasRootCauseInstanceOf(ReadError.class); - assertThatThrownBy(() -> p.parseStream(xif, CloseErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, CloseErrorStream::new, UTF_8)) .isInstanceOf(CloseError.class) .hasNoSuppressedExceptions() .hasNoCause(); @@ -69,22 +69,22 @@ public void testInvalidParseStream() throws IOException { } }; - assertThatNullPointerException().isThrownBy(() -> p.parseStream(null, EmptyStream::new, UTF_8)); + assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, null, EmptyStream::new, UTF_8)); - assertThatNullPointerException().isThrownBy(() -> p.parseStream(xif, null, UTF_8)); + assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, xif, null, UTF_8)); - assertThatThrownBy(() -> p.parseStream(xif, OpenErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, OpenErrorStream::new, UTF_8)) .isInstanceOf(OpenError.class) .hasNoSuppressedExceptions() .hasNoCause(); - assertThatThrownBy(() -> p.parseStream(xif, ReadErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, ReadErrorStream::new, UTF_8)) .isInstanceOf(Stax.XMLStreamIOException.class) .hasNoSuppressedExceptions() .hasCauseInstanceOf(XMLStreamException.class) .hasRootCauseInstanceOf(ReadError.class); - assertThatThrownBy(() -> p.parseStream(xif, CloseErrorStream::new, UTF_8)) + assertThatThrownBy(() -> Stax.parseStream(p, xif, CloseErrorStream::new, UTF_8)) .isInstanceOf(ParseError.class) .hasSuppressedException(new CloseError()) .hasNoCause(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java index feb33f30..2e7594bb 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java @@ -141,7 +141,7 @@ static IO.Function getDecoder(@Nullable String name) { .filter(o -> Objects.equals(name, o.name)) .map(o -> o.getDecoder()) .findAny() - .orElse(o -> o); + .orElse(IO.Function.identity()); } static String getEncodingHeader() { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java index a36b1b61..6c24e7e2 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java @@ -55,7 +55,7 @@ public final class Sdmx21RestClient implements RestClient { public List getFlows() throws IOException { URL url = getFlowsQuery(endpoint); return flow21(langs) - .asStreamParser(Stax.getInputFactory(), UTF_8) + .onInputStream(Stax.getInputFactory(), UTF_8) .parseWithIO(calling(url, XML)); } @@ -63,7 +63,7 @@ public List getFlows() throws IOException { public Dataflow getFlow(DataflowRef ref) throws IOException { URL url = getFlowQuery(endpoint, ref); return flow21(langs) - .asStreamParser(Stax.getInputFactory(), UTF_8) + .onInputStream(Stax.getInputFactory(), UTF_8) .parseWithIO(calling(url, XML)) .stream() .filter(ref::containsRef) @@ -75,7 +75,7 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { public DataStructure getStructure(DataStructureRef ref) throws IOException { URL url = getStructureQuery(endpoint, ref); return struct21(langs) - .asStreamParser(Stax.getInputFactory(), UTF_8) + .onInputStream(Stax.getInputFactory(), UTF_8) .parseWithIO(calling(url, STRUCTURE_21)) .stream() .filter(ref::equalsRef) @@ -87,7 +87,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { URL url = getDataQuery(endpoint, flowRef, query); return compactData21(dsd, dialect) - .asStreamParser(Stax.getInputFactoryWithoutNamespace(), UTF_8) + .onInputStream(Stax.getInputFactoryWithoutNamespace(), UTF_8) .parseWithIO(calling(url, STRUCTURE_SPECIFIC_DATA_21)); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java index ff7c58b1..47b98d13 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java @@ -43,7 +43,7 @@ public void assertDriverCompliance(SdmxWebDriver d, String prefix) { assertThat(d.getDefaultEntryPoints()) .allSatisfy(o -> checkEntryPoint(o, prefix)) - .allMatch(IO.asUnchecked(d::accepts)); + .allMatch(IO.Predicate.unchecked(d::accepts)); assertThat(d.getClass()).isFinal(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java index c49a9422..3c204c21 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java @@ -95,7 +95,7 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference ECB = new AtomicReference<>(); private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.struct20(l).onReader(f).parseWithIO(xml::openReader); } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -106,21 +106,21 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(f, xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).parseWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.struct21(l).onReader(f).parseWithIO(xml::openReader); } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).parseReader(f, xml::openReader); + return SdmxXmlStreams.flow21(l).onReader(f).parseWithIO(xml::openReader); } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(f, xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).parseWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } From 8038152fa3563690f9c5b37c090f9ff050053a2f Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 5 Dec 2017 10:43:26 +0100 Subject: [PATCH 60/68] Updated connectors dependency. --- sdmx-facade/sdmx-facade-web/pom.xml | 2 +- .../sdmx-facade-web/src/main/java/internal/connectors/Util.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/pom.xml b/sdmx-facade/sdmx-facade-web/pom.xml index 4566fb6b..0485369b 100644 --- a/sdmx-facade/sdmx-facade-web/pom.xml +++ b/sdmx-facade/sdmx-facade-web/pom.xml @@ -13,7 +13,7 @@ - 346366c9d7 + 5401d5ee08 RELEASE802 diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 6f4af4df..7b2141f7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -105,7 +105,7 @@ public DataFlowStructure fromStructure(DataStructure dsd) { result.setVersion(dsd.getRef().getVersion()); result.setName(dsd.getLabel()); result.setTimeDimension(dsd.getTimeDimensionId()); - // FIXME: how to set MEASURE? + result.setMeasure(dsd.getPrimaryMeasureId()); dsd.getDimensions().forEach(o -> result.setDimension(fromDimension(o))); return result; } From db60d2e39f4896d751e1adaf08db764319742dfc Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 6 Dec 2017 11:23:49 +0100 Subject: [PATCH 61/68] Moved IO util to its own project. --- .../demetra/sdmx/file/SdmxFileProvider.java | 2 +- .../nbb/demetra/sdmx/web/SdmxWebProvider.java | 2 +- .../java/internal/sdmx/SdmxCubeAccessor.java | 2 +- .../java/internal/sdmx/SdmxCubeItems.java | 2 +- .../java/test/samples/FacadeResource.java | 10 +- sdmx-facade/pom.xml | 9 +- .../internal/file/SdmxDecoderResource.java | 2 +- .../internal/file/xml/StaxSdmxDecoder.java | 6 +- .../file/xml/DataStructureDecoderTest.java | 8 +- .../internal/file/xml/DataTypeProbeTest.java | 12 +- sdmx-facade/sdmx-facade-samples/pom.xml | 4 + .../nbb/sdmx/facade/tck/ConnectionAssert.java | 6 +- .../nbb/sdmx/facade/tck/DataCursorAssert.java | 18 +- .../main/java/internal/io/ConsumerWithIO.java | 36 - sdmx-facade/sdmx-facade-util/pom.xml | 4 + .../nbb/sdmx/facade/util/SeriesSupport.java | 2 +- .../src/main/java/be/nbb/util/IO.java | 615 ------------------ .../src/main/java/be/nbb/util/Stax.java | 32 +- .../sdmx/facade/util/SeriesSupportTest.java | 8 +- .../xml/stream/XMLStreamFlow21Test.java | 2 +- .../xml/stream/XMLStreamStructure20Test.java | 7 +- .../xml/stream/XMLStreamStructure21Test.java | 4 +- .../src/test/java/be/nbb/util/IOTest.java | 599 ----------------- .../internal/web/sdmx21/RestExecutorImpl.java | 2 +- .../internal/web/sdmx21/Sdmx21RestClient.java | 10 +- .../internal/web/FailsafeRestClientTest.java | 2 +- .../src/test/java/test/DriverAssertions.java | 2 +- .../java/test/samples/FacadeResource.java | 10 +- 28 files changed, 87 insertions(+), 1331 deletions(-) delete mode 100644 sdmx-facade/sdmx-facade-samples/src/main/java/internal/io/ConsumerWithIO.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java delete mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 83df214b..d26a438e 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -24,7 +24,6 @@ import be.nbb.sdmx.facade.SdmxConnectionSupplier; import be.nbb.sdmx.facade.file.SdmxFileManager; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.util.IO; import com.google.common.cache.Cache; import ec.tss.ITsProvider; import ec.tss.tsproviders.DataSet; @@ -45,6 +44,7 @@ import internal.file.SdmxFileUtil; import internal.sdmx.SdmxCubeItems; import internal.sdmx.SdmxPropertiesSupport; +import ioutil.IO; import java.io.File; import java.io.IOException; import org.openide.util.lookup.ServiceProvider; diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java index 1e9626c0..afb9c332 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProvider.java @@ -22,7 +22,6 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.SdmxConnectionSupplier; -import be.nbb.util.IO; import be.nbb.sdmx.facade.web.SdmxWebManager; import com.google.common.cache.Cache; import ec.tss.ITsProvider; @@ -41,6 +40,7 @@ import ec.tstoolkit.utilities.GuavaCaches; import internal.sdmx.SdmxCubeItems; import internal.sdmx.SdmxPropertiesSupport; +import ioutil.IO; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; import org.openide.util.lookup.ServiceProvider; diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java index d3679576..a77ea9a6 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeAccessor.java @@ -21,7 +21,6 @@ import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import com.google.common.base.Converter; import com.google.common.collect.Maps; @@ -30,6 +29,7 @@ import ec.tss.tsproviders.cursor.TsCursor; import ec.tss.tsproviders.utils.IteratorWithIO; import ec.tstoolkit.design.VisibleForTesting; +import ioutil.IO; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java index 0d8c3d37..81343fa3 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxCubeItems.java @@ -22,13 +22,13 @@ import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.HasFilePaths; import ec.tss.tsproviders.cube.CubeAccessor; import ec.tss.tsproviders.cube.CubeId; import ec.tss.tsproviders.utils.IParam; +import ioutil.IO; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index 3c204c21..9ed7647f 100644 --- a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -95,7 +95,7 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference ECB = new AtomicReference<>(); private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.struct20(l).onReader(f).applyWithIO(xml::openReader); } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -106,21 +106,21 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).parseWithIO(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).applyWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.struct21(l).onReader(f).applyWithIO(xml::openReader); } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.flow21(l).onReader(f).applyWithIO(xml::openReader); } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).parseWithIO(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).applyWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index a8ed233f..b7e72332 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -20,12 +20,14 @@ UTF-8 1.8 1.8 + 0.7.9 + 3.0.2 1.16.18 4.12 3.8.0 - 0.7.9 + 0.0.1-SNAPSHOT @@ -51,6 +53,11 @@ assertj-core ${assertj.version} + + be.nbb.rd + java-io-util + ${java-io-util.version} + ${project.groupId} diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java index 77f62017..9f5d25f5 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java @@ -52,7 +52,7 @@ public SdmxDecoder.Info decode() throws IOException { public DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { return getDataSupplier(entry.getType(), entry.getStructure()) .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) - .parseWithIO(files.getData()); + .applyWithIO(files.getData()); } private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java index 44ffce94..81309c79 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java @@ -55,13 +55,13 @@ public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOExcep private DataType probeDataType(File data) throws IOException { return DataTypeProbe.of() .onFile(factory, StandardCharsets.UTF_8) - .parseWithIO(data); + .applyWithIO(data); } private DataStructure parseStruct(DataType dataType, LanguagePriorityList langs, File structure) throws IOException { return getStructParser(dataType, langs) .onFile(factory, StandardCharsets.UTF_8) - .parseWithIO(structure) + .applyWithIO(structure) .get(0); } @@ -81,7 +81,7 @@ private Stax.Parser> getStructParser(DataType o, LanguagePri private DataStructure decodeStruct(DataType dataType, File data) throws IOException { return getStructDecoder(dataType) .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) - .parseWithIO(data); + .applyWithIO(data); } private static Stax.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index 4f57512a..c2fdc0bc 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -45,7 +45,7 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic20().onReader(xif).parseWithIO(SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic20().onReader(xif).applyWithIO(SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); } @Test @@ -62,7 +62,7 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact20().onReader(xif).parseWithIO(SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact20().onReader(xif).applyWithIO(SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); } @Test @@ -81,7 +81,7 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic21().onReader(xif).parseWithIO(SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic21().onReader(xif).applyWithIO(SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); } @Test @@ -100,7 +100,7 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact21().onReader(xif).parseWithIO(SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact21().onReader(xif).applyWithIO(SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); } private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java index 25c11a97..856474a9 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java @@ -17,9 +17,9 @@ package internal.file.xml; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.util.IO; import be.nbb.util.Stax; import internal.file.SdmxDecoder; +import ioutil.IO; import java.io.IOException; import java.io.Reader; import org.junit.Test; @@ -33,27 +33,27 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { - assertThat(decoder.parseWithIO(SdmxSource.OTHER_GENERIC20::openReader)) + assertThat(decoder.applyWithIO(SdmxSource.OTHER_GENERIC20::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC20); } @Test public void testDecodeCompact20() throws IOException { - assertThat(decoder.parseWithIO(SdmxSource.OTHER_COMPACT20::openReader)) + assertThat(decoder.applyWithIO(SdmxSource.OTHER_COMPACT20::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT20); } @Test public void testDecodeGeneric21() throws IOException { - assertThat(decoder.parseWithIO(SdmxSource.OTHER_GENERIC21::openReader)) + assertThat(decoder.applyWithIO(SdmxSource.OTHER_GENERIC21::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC21); } @Test public void testDecodeCompact21() throws IOException { - assertThat(decoder.parseWithIO(SdmxSource.OTHER_COMPACT21::openReader)) + assertThat(decoder.applyWithIO(SdmxSource.OTHER_COMPACT21::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT21); } - private final IO.Parser, SdmxDecoder.DataType> decoder = DataTypeProbe.of().onReader(Stax.getInputFactory()); + private final IO.Function, SdmxDecoder.DataType> decoder = DataTypeProbe.of().onReader(Stax.getInputFactory()); } diff --git a/sdmx-facade/sdmx-facade-samples/pom.xml b/sdmx-facade/sdmx-facade-samples/pom.xml index d0c2bb8a..4511f7b5 100644 --- a/sdmx-facade/sdmx-facade-samples/pom.xml +++ b/sdmx-facade/sdmx-facade-samples/pom.xml @@ -24,6 +24,10 @@ provided + + be.nbb.rd + java-io-util + org.assertj assertj-core diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 934259f1..1b2091c4 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -17,13 +17,13 @@ package be.nbb.sdmx.facade.tck; import be.nbb.sdmx.facade.DataCursor; -import internal.io.ConsumerWithIO; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; +import ioutil.IO; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -96,10 +96,10 @@ private static void assertNonnull(SoftAssertions s, SdmxConnection conn, Dataflo .isInstanceOf(NullPointerException.class); } - private static void assertState(SoftAssertions s, Callable supplier, ConsumerWithIO consumer, String expression) throws Exception { + private static void assertState(SoftAssertions s, Callable supplier, IO.Consumer consumer, String expression) throws Exception { try (SdmxConnection conn = supplier.call()) { conn.close(); - s.assertThatThrownBy(() -> consumer.accept(conn)) + s.assertThatThrownBy(() -> consumer.acceptWithIO(conn)) .as("Expecting '%s' to raise IOException when called after close", expression) .isInstanceOf(IOException.class) .hasMessageContaining("closed"); diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/DataCursorAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/DataCursorAssert.java index 04306db4..1f24906a 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/DataCursorAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/DataCursorAssert.java @@ -16,8 +16,8 @@ */ package be.nbb.sdmx.facade.tck; -import internal.io.ConsumerWithIO; import be.nbb.sdmx.facade.DataCursor; +import ioutil.IO; import java.io.IOException; import java.util.concurrent.Callable; import org.assertj.core.api.SoftAssertions; @@ -62,7 +62,7 @@ public static void assertCompliance(SoftAssertions s, Callable suppl } try (DataCursor c = supplier.call()) { - nextSeriesToEnd().andThen(DataCursor::nextSeries).accept(c); + nextSeriesToEnd().andThen(DataCursor::nextSeries).acceptWithIO(c); } catch (Exception ex) { s.fail("Subsequent calls to #nextSeries must not raise exception", ex); } @@ -84,14 +84,14 @@ private static void assertNonnull(SoftAssertions s, DataCursor c) { s.assertThatThrownBy(() -> c.getSeriesAttribute(null)).isInstanceOf(NullPointerException.class); } - private static void assertState(SoftAssertions s, Callable supplier, ConsumerWithIO consumer, String method) { + private static void assertState(SoftAssertions s, Callable supplier, IO.Consumer consumer, String method) { s.assertThatThrownBy(() -> with(supplier, close().andThen(consumer))) .as("Calling %s after close must throw IOException", method) .isInstanceOf(IOException.class) .hasMessageContaining("closed"); } - private static void assertSeriesState(SoftAssertions s, Callable supplier, ConsumerWithIO consumer, String method) { + private static void assertSeriesState(SoftAssertions s, Callable supplier, IO.Consumer consumer, String method) { assertState(s, supplier, consumer, method); s.assertThatThrownBy(() -> with(supplier, consumer)) .as("Calling %s before first series must throw IllegalStateException", method) @@ -101,7 +101,7 @@ private static void assertSeriesState(SoftAssertions s, Callable sup .isInstanceOf(IllegalStateException.class); } - private static void assertObsState(SoftAssertions s, Callable supplier, ConsumerWithIO consumer, String method) throws Exception { + private static void assertObsState(SoftAssertions s, Callable supplier, IO.Consumer consumer, String method) throws Exception { assertSeriesState(s, supplier, consumer, method); try (DataCursor c = supplier.call()) { while (c.nextSeries()) { @@ -117,20 +117,20 @@ private static void assertObsState(SoftAssertions s, Callable suppli } } - private static void with(Callable supplier, ConsumerWithIO consumer) throws Exception { + private static void with(Callable supplier, IO.Consumer consumer) throws Exception { try (DataCursor c = supplier.call()) { - consumer.accept(c); + consumer.acceptWithIO(c); } } - static ConsumerWithIO nextSeriesToEnd() { + static IO.Consumer nextSeriesToEnd() { return c -> { while (c.nextSeries()) { } }; } - static ConsumerWithIO close() { + static IO.Consumer close() { return DataCursor::close; } } diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/internal/io/ConsumerWithIO.java b/sdmx-facade/sdmx-facade-samples/src/main/java/internal/io/ConsumerWithIO.java deleted file mode 100644 index 5a7ba1f4..00000000 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/internal/io/ConsumerWithIO.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.io; - -import java.io.IOException; - -/** - * - * @author Philippe Charles - * @param - */ -public interface ConsumerWithIO { - - void accept(T t) throws IOException; - - default ConsumerWithIO andThen(ConsumerWithIO after) { - return c -> { - accept(c); - after.accept(c); - }; - } -} diff --git a/sdmx-facade/sdmx-facade-util/pom.xml b/sdmx-facade/sdmx-facade-util/pom.xml index e7d6bcf8..0007ce57 100644 --- a/sdmx-facade/sdmx-facade-util/pom.xml +++ b/sdmx-facade/sdmx-facade-util/pom.xml @@ -28,6 +28,10 @@ ${project.groupId} sdmx-facade-api + + be.nbb.rd + java-io-util + junit diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index 8358c786..312df544 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -16,12 +16,12 @@ */ package be.nbb.sdmx.facade.util; -import be.nbb.util.IO; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.Series; +import ioutil.IO; import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java deleted file mode 100644 index d3a58c5a..00000000 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/IO.java +++ /dev/null @@ -1,615 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.util; - -import java.io.Closeable; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.Iterator; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.StreamSupport; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** - * - * @author Philippe Charles - */ -@lombok.experimental.UtilityClass -public class IO { - - /** - * Represents a function without argument and result. - */ - @FunctionalInterface - public interface Runnable { - - /** - * Run this function. - * - * @throws IOException - */ - @JdkWithIO - void runWithIO() throws IOException; - - @Nonnull - default Closeable asCloseable() { - return this::runWithIO; - } - - @Nonnull - default java.lang.Runnable asUnchecked() { - return () -> { - try { - runWithIO(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - @Nonnull - static java.lang.Runnable unchecked(@Nonnull Runnable o) { - return o.asUnchecked(); - } - - @Nonnull - static Runnable checked(@Nonnull java.lang.Runnable o) { - return () -> { - try { - o.run(); - } catch (UncheckedIOException e) { - throw e.getCause(); - } - }; - } - - @Nonnull - static Runnable throwing(@Nonnull java.util.function.Supplier ex) { - Objects.requireNonNull(ex); - return () -> { - throw ex.get(); - }; - } - - @Nonnull - static Runnable noOp() { - return () -> { - }; - } - } - - /** - * Represents a supplier of results. - * - * @param the type of results supplied by this supplier - */ - @FunctionalInterface - public interface Supplier { - - /** - * Gets a result. - * - * @return a result - * @throws IOException - */ - @JdkWithIO - T getWithIO() throws IOException; - - @Nonnull - default java.util.function.Supplier asUnchecked() { - return () -> { - try { - return getWithIO(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - @Nonnull - static java.util.function.Supplier unchecked(@Nonnull Supplier o) { - return o.asUnchecked(); - } - - @Nonnull - static Supplier checked(@Nonnull java.util.function.Supplier o) { - Objects.requireNonNull(o); - return () -> { - try { - return o.get(); - } catch (UncheckedIOException e) { - throw e.getCause(); - } - }; - } - - @Nonnull - static Supplier throwing(@Nonnull java.util.function.Supplier ex) { - Objects.requireNonNull(ex); - return () -> { - throw ex.get(); - }; - } - - @Nonnull - @SuppressWarnings("null") - static Supplier of(@Nullable T t) { - return () -> t; - } - } - - /** - * Represents a function that accepts one argument and produces a result. - * - * @param the type of the input to the function - * @param the type of the result of the function - */ - @FunctionalInterface - public interface Function { - - /** - * Applies this function to the given argument. - * - * @param t the function argument - * @return the function result - * @throws java.io.IOException - */ - @JdkWithIO - R applyWithIO(T t) throws IOException; - - /** - * Returns a composed function that first applies the {@code before} - * function to its input, and then applies this function to the result. - * If evaluation of either function throws an exception, it is relayed - * to the caller of the composed function. - * - * @param the type of input to the {@code before} function, and to - * the composed function - * @param before the function to apply before this function is applied - * @return a composed function that first applies the {@code before} - * function and then applies this function - * @throws NullPointerException if before is null - * - * @see #andThen(FunctionWithIO) - */ - @JdkWithIO - @Nonnull - default Function compose(@Nonnull Function before) { - Objects.requireNonNull(before); - return (V v) -> applyWithIO(before.applyWithIO(v)); - } - - /** - * Returns a composed function that first applies this function to its - * input, and then applies the {@code after} function to the result. If - * evaluation of either function throws an exception, it is relayed to - * the caller of the composed function. - * - * @param the type of output of the {@code after} function, and of - * the composed function - * @param after the function to apply after this function is applied - * @return a composed function that first applies this function and then - * applies the {@code after} function - * @throws NullPointerException if after is null - * - * @see #compose(FunctionWithIO) - */ - @JdkWithIO - @Nonnull - default Function andThen(@Nonnull Function after) { - Objects.requireNonNull(after); - return (T t) -> after.applyWithIO(applyWithIO(t)); - } - - @Nonnull - default java.util.function.Function asUnchecked() { - return (T t) -> { - try { - return applyWithIO(t); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - @Nonnull - static java.util.function.Function unchecked(@Nonnull Function o) { - return o.asUnchecked(); - } - - @Nonnull - static Function checked(@Nonnull java.util.function.Function func) { - Objects.requireNonNull(func); - return o -> { - try { - return func.apply(o); - } catch (UncheckedIOException ex) { - throw ex.getCause(); - } - }; - } - - @Nonnull - static Function throwing(@Nonnull java.util.function.Supplier ex) { - Objects.requireNonNull(ex); - return o -> { - throw ex.get(); - }; - } - - /** - * Returns a function that always returns its input argument. - * - * @param the type of the input and output objects to the function - * @return a function that always returns its input argument - */ - @JdkWithIO - @Nonnull - static Function identity() { - return t -> t; - } - - @Nonnull - @SuppressWarnings("null") - static Function of(@Nullable R r) { - return o -> r; - } - } - - /** - * Represents a predicate (boolean-valued function) of one argument. - * - * @param the type of the input to the predicate - */ - @FunctionalInterface - public interface Predicate { - - /** - * Evaluates this predicate on the given argument. - * - * @param t the input argument - * @return {@code true} if the input argument matches the predicate, - * otherwise {@code false} - * @throws java.io.IOException - */ - @JdkWithIO - boolean testWithIO(T t) throws IOException; - - /** - * Returns a composed predicate that represents a short-circuiting - * logical AND of this predicate and another. When evaluating the - * composed predicate, if this predicate is {@code false}, then the - * {@code other} predicate is not evaluated. - * - *

- * Any exceptions thrown during evaluation of either predicate are - * relayed to the caller; if evaluation of this predicate throws an - * exception, the {@code other} predicate will not be evaluated. - * - * @param other a predicate that will be logically-ANDed with this - * predicate - * @return a composed predicate that represents the short-circuiting - * logical AND of this predicate and the {@code other} predicate - * @throws NullPointerException if other is null - */ - @JdkWithIO - @Nonnull - default Predicate and(@Nonnull Predicate other) { - Objects.requireNonNull(other); - return (t) -> testWithIO(t) && other.testWithIO(t); - } - - /** - * Returns a predicate that represents the logical negation of this - * predicate. - * - * @return a predicate that represents the logical negation of this - * predicate - */ - @JdkWithIO - @Nonnull - default Predicate negate() { - return (t) -> !testWithIO(t); - } - - /** - * Returns a composed predicate that represents a short-circuiting - * logical OR of this predicate and another. When evaluating the - * composed predicate, if this predicate is {@code true}, then the - * {@code other} predicate is not evaluated. - * - *

- * Any exceptions thrown during evaluation of either predicate are - * relayed to the caller; if evaluation of this predicate throws an - * exception, the {@code other} predicate will not be evaluated. - * - * @param other a predicate that will be logically-ORed with this - * predicate - * @return a composed predicate that represents the short-circuiting - * logical OR of this predicate and the {@code other} predicate - * @throws NullPointerException if other is null - */ - @JdkWithIO - @Nonnull - default Predicate or(@Nonnull Predicate other) { - Objects.requireNonNull(other); - return (t) -> testWithIO(t) || other.testWithIO(t); - } - - @Nonnull - default java.util.function.Predicate asUnchecked() { - return (T t) -> { - try { - return testWithIO(t); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - @Nonnull - static java.util.function.Predicate unchecked(@Nonnull Predicate o) { - return o.asUnchecked(); - } - - @Nonnull - static Predicate checked(@Nonnull java.util.function.Predicate predicate) { - Objects.requireNonNull(predicate); - return o -> { - try { - return predicate.test(o); - } catch (UncheckedIOException ex) { - throw ex.getCause(); - } - }; - } - - @Nonnull - static Predicate throwing(@Nonnull java.util.function.Supplier ex) { - Objects.requireNonNull(ex); - return o -> { - throw ex.get(); - }; - } - - /** - * Returns a predicate that tests if two arguments are equal according - * to {@link Objects#equals(Object, Object)}. - * - * @param the type of arguments to the predicate - * @param targetRef the object reference with which to compare for - * equality, which may be {@code null} - * @return a predicate that tests if two arguments are equal according - * to {@link Objects#equals(Object, Object)} - */ - @JdkWithIO - @Nonnull - static Predicate isEqual(Object targetRef) { - return (null == targetRef) - ? Objects::isNull - : object -> targetRef.equals(object); - } - - @Nonnull - static Predicate of(boolean r) { - return o -> r; - } - } - - /** - * Represents an operation that accepts a single input argument and returns - * no result. Unlike most other functional interfaces, {@code Consumer} is - * expected to operate via side-effects. - * - * @param the type of the input to the operation - */ - @FunctionalInterface - public interface Consumer { - - /** - * Performs this operation on the given argument. - * - * @param t the input argument - * @throws java.io.IOException - */ - @JdkWithIO - void acceptWithIO(T t) throws IOException; - - /** - * Returns a composed {@code Consumer} that performs, in sequence, this - * operation followed by the {@code after} operation. If performing - * either operation throws an exception, it is relayed to the caller of - * the composed operation. If performing this operation throws an - * exception, the {@code after} operation will not be performed. - * - * @param after the operation to perform after this operation - * @return a composed {@code Consumer} that performs in sequence this - * operation followed by the {@code after} operation - * @throws NullPointerException if {@code after} is null - */ - @JdkWithIO - @Nonnull - default Consumer andThen(@Nonnull Consumer after) { - Objects.requireNonNull(after); - return (T t) -> { - acceptWithIO(t); - after.acceptWithIO(t); - }; - } - - @Nonnull - default java.util.function.Consumer asUnchecked() { - return (T t) -> { - try { - acceptWithIO(t); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - @Nonnull - static java.util.function.Consumer unchecked(@Nonnull Consumer o) { - return o.asUnchecked(); - } - - @Nonnull - static Consumer checked(@Nonnull java.util.function.Consumer consumer) { - return o -> { - try { - consumer.accept(o); - } catch (UncheckedIOException ex) { - throw ex.getCause(); - } - }; - } - - @Nonnull - static Consumer throwing(@Nonnull java.util.function.Supplier ex) { - Objects.requireNonNull(ex); - return o -> { - throw ex.get(); - }; - } - - @Nonnull - static Consumer noOp() { - return o -> { - }; - } - } - - @FunctionalInterface - public interface Parser { - - R parseWithIO(@Nonnull T source) throws IOException; - - @Nonnull - static Parser valueOf( - @Nonnull Function opener, - @Nonnull Function reader, - @Nonnull Consumer closer) { - Objects.requireNonNull(opener); - Objects.requireNonNull(reader); - Objects.requireNonNull(closer); - return source -> { - R resource = opener.applyWithIO(source); - try (Closeable c = () -> closer.acceptWithIO(resource)) { - return reader.applyWithIO(resource); - } - }; - } - - @Nonnull - static Parser flowOf( - @Nonnull Function opener, - @Nonnull Function reader, - @Nonnull Consumer closer) { - Objects.requireNonNull(opener); - Objects.requireNonNull(reader); - Objects.requireNonNull(closer); - return source -> { - R resource = opener.applyWithIO(source); - try { - return reader.applyWithIO(resource); - } catch (Error | RuntimeException | IOException e) { - try { - closer.acceptWithIO(resource); - } catch (IOException ex) { - try { - e.addSuppressed(ex); - } catch (Throwable ignore) { - } - } - throw e; - } - }; - } - } - - @lombok.experimental.UtilityClass - public static final class Stream { - - @Nonnull - public java.util.stream.Stream open(@Nonnull Supplier source, @Nonnull Function> streamer) throws IOException { - return asParser(streamer).parseWithIO(source); - } - - @Nonnull - public java.util.stream.Stream generateUntilNull(@Nonnull Supplier generator) { - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(asIterator(generator), Spliterator.ORDERED | Spliterator.NONNULL), false); - } - - private Parser, java.util.stream.Stream> asParser(Function> streamer) { - return Parser.flowOf( - source -> source.getWithIO(), - resource -> streamer.applyWithIO(resource).onClose(Runnable.unchecked(resource::close)), - Closeable::close - ); - } - - @Nonnull - private Iterator asIterator(@Nonnull Supplier nextSupplier) { - Objects.requireNonNull(nextSupplier); - return new Iterator() { - T nextElement = null; - - @Override - public boolean hasNext() { - if (nextElement != null) { - return true; - } else { - try { - nextElement = nextSupplier.getWithIO(); - return (nextElement != null); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - - @Override - public T next() { - if (nextElement != null || hasNext()) { - T line = nextElement; - nextElement = null; - return line; - } else { - throw new NoSuchElementException(); - } - } - }; - } - } - - @Target({ElementType.METHOD}) - @Retention(RetentionPolicy.SOURCE) - @Documented - private @interface JdkWithIO { - - String value() default ""; - } -} diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java index 2b09cd54..ba069270 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java @@ -16,6 +16,8 @@ */ package be.nbb.util; +import ioutil.IO; +import ioutil.Xml; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; @@ -25,6 +27,7 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import javax.annotation.Nonnull; import javax.xml.stream.EventFilter; import javax.xml.stream.StreamFilter; @@ -63,28 +66,27 @@ public interface Parser { T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; @Nonnull - default IO.Parser onFile(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + default IO.Function onFile(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { return source -> parseStream(this, xf, () -> new FileInputStream(source), cs); } @Nonnull - default IO.Parser onPath(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + default IO.Function onPath(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { return source -> { - try { - return parseStream(this, xf, () -> new FileInputStream(source.toFile()), cs); - } catch (UnsupportedOperationException ex) { - return parseReader(this, xf, () -> Files.newBufferedReader(source, cs)); - } + Optional file = IO.getFile(source); + return file.isPresent() + ? parseStream(this, xf, () -> new FileInputStream(file.get()), cs) + : parseReader(this, xf, () -> Files.newBufferedReader(source, cs)); }; } @Nonnull - default IO.Parser, T> onInputStream(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { + default IO.Function, T> onInputStream(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { return source -> parseStream(this, xf, source, cs); } @Nonnull - default IO.Parser, T> onReader(@Nonnull XMLInputFactory xf) { + default IO.Function, T> onReader(@Nonnull XMLInputFactory xf) { return source -> parseReader(this, xf, source); } @@ -166,16 +168,6 @@ public boolean isNotNamespaceAware(@Nonnull XMLStreamReader f) { return !(Boolean) f.getProperty(XMLInputFactory.IS_NAMESPACE_AWARE); } - // https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Prevention_Cheat_Sheet#XMLInputFactory_.28a_StAX_parser.29 - public static void preventXXE(@Nonnull XMLInputFactory factory) { - if (factory.isPropertySupported(XMLInputFactory.SUPPORT_DTD)) { - factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - } - if (factory.isPropertySupported(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES)) { - factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - } - } - private static final class ImmutableInputFactory extends XMLInputFactory { static final XMLInputFactory DEFAULT = new ImmutableInputFactory(true); @@ -188,7 +180,7 @@ private ImmutableInputFactory(boolean namespaceAware) { if (!namespaceAware && delegate.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE)) { delegate.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); } - preventXXE(delegate); + Xml.StAX.preventXXE(delegate); } @Override diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index b1a12901..10cbb5e6 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -76,8 +76,8 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).parseWithIO(SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).applyWithIO(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } } @@ -99,8 +99,8 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).parseWithIO(SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).applyWithIO(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(120); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java index e401bbb3..bd8d82ca 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java @@ -38,7 +38,7 @@ public class XMLStreamFlow21Test { public void test() throws IOException { Stax.Parser> p = Stax.Parser.of(new XMLStreamFlow21(LanguagePriorityList.ANY)::parse); - assertThat(p.onReader(xif).parseWithIO(SdmxSource.ECB_DATAFLOWS::openReader)) + assertThat(p.onReader(xif).applyWithIO(SdmxSource.ECB_DATAFLOWS::openReader)) .containsExactly( Dataflow.of(DataflowRef.of("ECB", "AME", "1.0"), DataStructureRef.of("ECB", "ECB_AME1", "1.0"), "AMECO"), Dataflow.of(DataflowRef.of("ECB", "BKN", "1.0"), DataStructureRef.of("ECB", "ECB_BKN1", "1.0"), "Banknotes statistics"), diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index f68ae4a1..357b06e0 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -21,7 +21,6 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; -import java.io.IOException; import java.util.List; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; @@ -39,7 +38,7 @@ public class XMLStreamStructure20Test { public void test() throws Exception { Stax.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - assertThat(p1.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p1.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -53,7 +52,7 @@ public void test() throws Exception { Stax.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - assertThat(p2.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p2.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -66,7 +65,7 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> p1.onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> p1.onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index c7952370..ba795cca 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -38,7 +38,7 @@ public class XMLStreamStructure21Test { public void test() throws Exception { Stax.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - assertThat(parser.onReader(xif).parseWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(parser.onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); @@ -51,7 +51,7 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> parser.onReader(xif).parseWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> parser.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java deleted file mode 100644 index 7fcb0d9e..00000000 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/IOTest.java +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.util; - -import java.io.Closeable; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import static org.assertj.core.api.Assertions.*; -import org.junit.Test; - -/** - * - * @author Philippe Charles - */ -public class IOTest { - - // - private static IO.Function> streamerOf(R... values) { - return o -> Stream.of(values); - } - - @Test - @SuppressWarnings("null") - public void testStreamOpen() throws IOException { - assertThatNullPointerException().isThrownBy(() -> IO.Stream.open(null, streamerOf())); - assertThatNullPointerException().isThrownBy(() -> IO.Stream.open(() -> null, null)); - - IO.Supplier ofOpenError = IO.Supplier.throwing(OpenError::new); - IO.Supplier ofCloseError = () -> IO.Runnable.throwing(CloseError::new)::runWithIO; - IO.Function> toReadError = IO.Function.throwing(ReadError::new); - - assertThatThrownBy(() -> IO.Stream.open(ofOpenError, streamerOf()).close()) - .isInstanceOf(OpenError.class) - .hasNoSuppressedExceptions(); - - assertThatThrownBy(() -> IO.Stream.open(ofOpenError, toReadError).close()) - .isInstanceOf(OpenError.class) - .hasNoSuppressedExceptions(); - - assertThatThrownBy(() -> IO.Stream.open(ofCloseError, streamerOf()).close()) - .isInstanceOf(UncheckedIOException.class) - .hasRootCauseInstanceOf(CloseError.class) - .hasNoSuppressedExceptions(); - - assertThatThrownBy(() -> IO.Stream.open(ofCloseError, toReadError).close()) - .isInstanceOf(ReadError.class) - .hasSuppressedException(new CloseError()); - - assertThat(new AtomicInteger(0)).satisfies(c -> { - assertThatThrownBy(() -> IO.Stream.open(() -> c::incrementAndGet, toReadError).close()) - .isInstanceOf(ReadError.class) - .hasNoSuppressedExceptions(); - assertThat(c).hasValue(1); - }); - - assertThat(new AtomicInteger(0)).satisfies(c -> { - assertThatCode(() -> IO.Stream.open(() -> c::incrementAndGet, streamerOf()).close()).doesNotThrowAnyException(); - assertThat(c).hasValue(1); - }); - - assertThat(new AtomicInteger(0)).satisfies(c -> { - assertThatCode(() -> IO.Stream.open(() -> c::incrementAndGet, streamerOf())).doesNotThrowAnyException(); - assertThat(c).hasValue(0); - }); - - assertThat(IO.Stream.open(IO.Runnable.noOp()::asCloseable, streamerOf())).isEmpty(); - assertThat(IO.Stream.open(IO.Runnable.noOp()::asCloseable, streamerOf("a", "b", "c"))).hasSize(3); - } - - @Test - @SuppressWarnings("null") - public void testStreamGenerateUntilNull() { - assertThatNullPointerException().isThrownBy(() -> IO.Stream.generateUntilNull(null)); - - assertThat(IO.Stream.generateUntilNull(() -> null)).isEmpty(); - - Iterator iter = Arrays.asList("A", "B").iterator(); - assertThat(IO.Stream.generateUntilNull(() -> iter.hasNext() ? iter.next() : null)).containsExactly("A", "B"); - - assertThatThrownBy(() -> IO.Stream.generateUntilNull(ofError1).count()) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - // - - // - private final IO.Function toUpperCase = String::toUpperCase; - private final IO.Function toString = Object::toString; - private final IO.Function toError1 = IO.Function.throwing(Error1::new); - private final IO.Function toError2 = IO.Function.throwing(Error2::new); - - @Test - @SuppressWarnings("null") - public void testFunctionCompose() throws IOException { - assertThatNullPointerException().isThrownBy(() -> toUpperCase.compose(null)); - - assertThat(toUpperCase.compose(toString).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString().toUpperCase()); - assertThatThrownBy(() -> toError1.compose(toString).applyWithIO(Byte.class)).isInstanceOf(Error1.class); - assertThatThrownBy(() -> toString.compose(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); - assertThatThrownBy(() -> toError1.compose(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); - } - - @Test - @SuppressWarnings("null") - public void testFunctionAndThen() throws IOException { - assertThatNullPointerException().isThrownBy(() -> toUpperCase.andThen(null)); - - assertThat(toString.andThen(toUpperCase).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString().toUpperCase()); - assertThatThrownBy(() -> toError1.andThen(toString).applyWithIO(Byte.class)).isInstanceOf(Error1.class); - assertThatThrownBy(() -> toString.andThen(toError2).applyWithIO(Byte.class)).isInstanceOf(Error2.class); - assertThatThrownBy(() -> toError1.andThen(toError2).applyWithIO(Byte.class)).isInstanceOf(Error1.class); - } - - @Test - public void testFunctionAsUnchecked() { - assertThat(toUpperCase.asUnchecked().apply("hello")).isEqualTo("HELLO"); - assertThatThrownBy(() -> toError1.asUnchecked().apply(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testFunctionUnchecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Function.unchecked(null)); - - assertThat(IO.Function.unchecked(toUpperCase).apply("hello")).isEqualTo("HELLO"); - assertThatThrownBy(() -> IO.Function.unchecked(toError1).apply(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testFunctionChecked() throws IOException { - assertThatNullPointerException().isThrownBy(() -> IO.Function.checked(null)); - - assertThat(IO.Function.checked(Object::toString).applyWithIO(Byte.class)).isEqualTo(Byte.class.toString()); - assertThatThrownBy(() -> IO.Function.checked(IO.Function.unchecked(toError1)).applyWithIO(null)) - .isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testFunctionThrowing() { - assertThatNullPointerException().isThrownBy(() -> IO.Function.throwing(null)); - - assertThatThrownBy(() -> IO.Function.throwing(Error1::new).applyWithIO(null)).isInstanceOf(Error1.class); - } - - @Test - public void testFunctionIdentity() throws IOException { - assertThat(IO.Function.identity().applyWithIO(null)).isNull(); - assertThat(IO.Function.identity().applyWithIO(Byte.class)).isEqualTo(Byte.class); - } - - @Test - public void testFunctionOf() throws IOException { - assertThat(IO.Function.of(null).applyWithIO(null)).isNull(); - assertThat(IO.Function.of("").applyWithIO(Byte.class)).isEqualTo(""); - } - // - - // - private final IO.Runnable onError1 = IO.Runnable.throwing(Error1::new); - - @Test - public void testRunnableAsCloseable() { - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatCode(() -> ((IO.Runnable) o::incrementAndGet).asCloseable().close()).doesNotThrowAnyException(); - assertThat(o).hasValue(1); - }); - - assertThatThrownBy(() -> onError1.asCloseable().close()).isInstanceOf(Error1.class); - } - - @Test - public void testRunnableAsUnchecked() { - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatCode(() -> ((IO.Runnable) o::incrementAndGet).asUnchecked().run()).doesNotThrowAnyException(); - assertThat(o).hasValue(1); - }); - - assertThatThrownBy(() -> onError1.asUnchecked().run()) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testRunnableUnchecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Runnable.unchecked(null)); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatCode(() -> IO.Runnable.unchecked(o::incrementAndGet).run()).doesNotThrowAnyException(); - assertThat(o).hasValue(1); - }); - - assertThatThrownBy(() -> IO.Runnable.unchecked(onError1).run()) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testRunnableChecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Runnable.unchecked(null)); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatCode(() -> IO.Runnable.checked(IO.Runnable.unchecked(o::incrementAndGet)).runWithIO()).doesNotThrowAnyException(); - assertThat(o).hasValue(1); - }); - - assertThatThrownBy(() -> IO.Runnable.checked(IO.Runnable.unchecked(onError1)).runWithIO()).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testRunnableThrowing() { - assertThatNullPointerException().isThrownBy(() -> IO.Runnable.throwing(null)); - - assertThatThrownBy(() -> onError1.runWithIO()).isInstanceOf(Error1.class); - } - - @Test - public void testRunnableNoOp() { - assertThatCode(() -> IO.Runnable.noOp().runWithIO()).doesNotThrowAnyException(); - } - // - - // - private final IO.Supplier ofError1 = IO.Supplier.throwing(Error1::new); - private final IO.Supplier ofIncrement = new AtomicInteger()::incrementAndGet; - - @Test - public void testSupplierAsUnchecked() throws IOException { - assertThat(ofIncrement.asUnchecked().get()).isEqualTo(ofIncrement.getWithIO() - 1); - assertThatThrownBy(() -> ofError1.asUnchecked().get()) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testSupplierUnchecked() throws IOException { - assertThatNullPointerException().isThrownBy(() -> IO.Supplier.unchecked(null)); - - assertThat(IO.Supplier.unchecked(ofIncrement).get()).isEqualTo(ofIncrement.getWithIO() - 1); - assertThatThrownBy(() -> IO.Supplier.unchecked(ofError1).get()) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testSupplierChecked() throws IOException { - assertThatNullPointerException().isThrownBy(() -> IO.Supplier.checked(null)); - - assertThat(IO.Supplier.checked(IO.Supplier.unchecked(ofIncrement)).getWithIO()).isEqualTo(ofIncrement.getWithIO() - 1); - assertThatThrownBy(() -> IO.Supplier.checked(IO.Supplier.unchecked(ofError1)).getWithIO()).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testSupplierThrowing() { - assertThatNullPointerException().isThrownBy(() -> IO.Supplier.throwing(null)); - assertThatThrownBy(() -> ofError1.getWithIO()).isInstanceOf(Error1.class); - } - - @Test - public void testSupplierOf() throws IOException { - assertThat(IO.Supplier.of(null).getWithIO()).isNull(); - assertThat(IO.Supplier.of("").getWithIO()).isEqualTo(""); - } - // - - // - private final IO.Predicate isNonNull = Objects::nonNull; - private final IO.Predicate isNull = Objects::isNull; - private final IO.Predicate isNotEmptyString = o -> ((String) o).length() > 0; - private final IO.Predicate isError1 = IO.Predicate.throwing(Error1::new); - private final IO.Predicate isError2 = IO.Predicate.throwing(Error2::new); - - @Test - @SuppressWarnings("null") - public void testPredicateAnd() throws IOException { - assertThatNullPointerException().isThrownBy(() -> isNonNull.and(null)); - - assertThat(isNonNull.and(isNotEmptyString).testWithIO("a")).isTrue(); - assertThat(isNonNull.and(isNotEmptyString).testWithIO("")).isFalse(); - assertThat(isNonNull.and(isNotEmptyString).testWithIO(null)).isFalse(); - assertThatThrownBy(() -> isNonNull.and(isError1).testWithIO("")).isInstanceOf(Error1.class); - assertThatThrownBy(() -> isError1.and(isNonNull).testWithIO("")).isInstanceOf(Error1.class); - assertThatThrownBy(() -> isError1.and(isError2).testWithIO("")).isInstanceOf(Error1.class); - } - - @Test - public void testPredicateNegate() throws IOException { - assertThat(isNonNull.negate().testWithIO(null)).isTrue(); - assertThat(isNonNull.negate().testWithIO("")).isFalse(); - assertThatThrownBy(() -> isError1.negate().testWithIO("")).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testPredicateOr() throws IOException { - assertThatNullPointerException().isThrownBy(() -> isNonNull.or(null)); - - assertThat(isNull.or(isNotEmptyString).testWithIO("a")).isTrue(); - assertThat(isNull.or(isNotEmptyString).testWithIO("")).isFalse(); - assertThat(isNull.or(isNotEmptyString).testWithIO(null)).isTrue(); - assertThat(isNull.or(isError1).testWithIO(null)).isTrue(); - assertThatThrownBy(() -> isNull.or(isError1).testWithIO("")).isInstanceOf(Error1.class); - assertThatThrownBy(() -> isError1.or(isNull).testWithIO("")).isInstanceOf(Error1.class); - assertThatThrownBy(() -> isError1.or(isError2).testWithIO("")).isInstanceOf(Error1.class); - } - - @Test - public void testPredicateAsUnchecked() { - assertThat(IO.Predicate.of(true).asUnchecked().test("")).isTrue(); - assertThatThrownBy(() -> isError1.asUnchecked().test(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testPredicateUnchecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Predicate.unchecked(null)); - - assertThat(IO.Predicate.unchecked(IO.Predicate.of(true)).test("")).isTrue(); - assertThatThrownBy(() -> IO.Predicate.unchecked(isError1).test(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testPredicateChecked() throws IOException { - assertThatNullPointerException().isThrownBy(() -> IO.Predicate.checked(null)); - - assertThat(IO.Predicate.checked(o -> true).testWithIO("")).isTrue(); - assertThatThrownBy(() -> IO.Predicate.checked(IO.Predicate.unchecked(isError1)).testWithIO(null)).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testPredicateThrowing() { - assertThatNullPointerException().isThrownBy(() -> IO.Predicate.throwing(null)); - - assertThatThrownBy(() -> isError1.testWithIO(null)).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testPredicateIsEqual() throws IOException { - assertThat(IO.Predicate.isEqual(null).testWithIO(null)).isTrue(); - assertThat(IO.Predicate.isEqual(null).testWithIO("")).isFalse(); - assertThat(IO.Predicate.isEqual("").testWithIO("")).isTrue(); - assertThat(IO.Predicate.isEqual("").testWithIO(null)).isFalse(); - } - - @Test - public void testPredicateOf() throws IOException { - assertThat(IO.Predicate.of(true).testWithIO(null)).isTrue(); - assertThat(IO.Predicate.of(false).testWithIO("")).isFalse(); - } - // - - // - private final IO.Consumer withError1 = IO.Consumer.throwing(Error1::new); - private final IO.Consumer withError2 = IO.Consumer.throwing(Error2::new); - private final IO.Consumer withIncrement = AtomicInteger::incrementAndGet; - - @Test - @SuppressWarnings("null") - public void testConsumerAndThen() { - assertThatNullPointerException().isThrownBy(() -> withIncrement.andThen(null)); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatCode(() -> withIncrement.andThen(withIncrement).acceptWithIO(o)).doesNotThrowAnyException(); - assertThat(o.get()).isEqualTo(2); - }); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatThrownBy(() -> withError1.andThen(withIncrement).acceptWithIO(o)).isInstanceOf(Error1.class); - assertThat(o.get()).isEqualTo(0); - }); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatThrownBy(() -> withIncrement.andThen(withError1).acceptWithIO(o)).isInstanceOf(Error1.class); - assertThat(o.get()).isEqualTo(1); - }); - - assertThat(new AtomicInteger(0)).satisfies(o -> { - assertThatThrownBy(() -> withError1.andThen(withError2).acceptWithIO(o)).isInstanceOf(Error1.class); - assertThat(o.get()).isEqualTo(0); - }); - } - - @Test - public void testConsumerAsUnchecked() { - assertThatCode(() -> IO.Consumer.noOp().asUnchecked().accept(null)).doesNotThrowAnyException(); - assertThatThrownBy(() -> withError1.asUnchecked().accept(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testConsumerUnchecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Consumer.unchecked(null)); - - assertThatCode(() -> IO.Consumer.unchecked(IO.Consumer.noOp()).accept(null)).doesNotThrowAnyException(); - assertThatThrownBy(() -> IO.Consumer.unchecked(withError1).accept(null)) - .isInstanceOf(UncheckedIOException.class) - .hasCauseExactlyInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testConsumerChecked() { - assertThatNullPointerException().isThrownBy(() -> IO.Consumer.unchecked(null)); - - assertThatCode(() -> IO.Consumer.checked(IO.Consumer.unchecked(IO.Consumer.noOp())).acceptWithIO(null)).doesNotThrowAnyException(); - assertThatThrownBy(() -> IO.Consumer.checked(IO.Consumer.unchecked(withError1)).acceptWithIO(null)).isInstanceOf(Error1.class); - } - - @Test - @SuppressWarnings("null") - public void testConsumerThrowing() { - assertThatNullPointerException().isThrownBy(() -> IO.Consumer.throwing(null)); - - assertThatThrownBy(() -> IO.Consumer.throwing(Error1::new).acceptWithIO(null)).isInstanceOf(Error1.class); - } - - @Test - public void testConsumerNoOp() { - assertThatCode(() -> IO.Consumer.noOp().acceptWithIO(null)).doesNotThrowAnyException(); - } - // - - @lombok.AllArgsConstructor - private static final class X implements Closeable { - - private final List events; - - static X open(List stack) throws IOException { - X result = new X(stack); - result.events.add("open"); - return result; - } - - public Closeable read() throws IOException { - events.add("read"); - return this; - } - - @Override - public void close() throws IOException { - events.add("close"); - } - } - - @Test - public void testParserValueOf() throws IOException { - IO.Function openError = IO.Function.throwing(OpenError::new); - IO.Function readError = IO.Function.throwing(ReadError::new); - IO.Consumer closeError = IO.Consumer.throwing(CloseError::new); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Object> p = IO.Parser.valueOf(X::open, X::read, X::close); - assertThatCode(() -> p.parseWithIO((List) c)).doesNotThrowAnyException(); - assertThat(c).containsExactly("open", "read", "close"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Object> p = IO.Parser.valueOf(openError, X::read, X::close); - assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(OpenError.class); - assertThat(c).isEmpty(); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Object> p = IO.Parser.valueOf(X::open, readError, X::close); - assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(ReadError.class); - assertThat(c).containsExactly("open", "close"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Object> p = IO.Parser.valueOf(X::open, X::read, closeError); - assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(CloseError.class); - assertThat(c).containsExactly("open", "read"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Object> p = IO.Parser.valueOf(X::open, readError, closeError); - assertThatThrownBy(() -> p.parseWithIO((List) c)).isInstanceOf(ReadError.class).hasSuppressedException(new CloseError()); - assertThat(c).containsExactly("open"); - }); - } - - @Test - public void testParserFlowOf() throws IOException { - IO.Function openError = IO.Function.throwing(OpenError::new); - IO.Function readError = IO.Function.throwing(ReadError::new); - IO.Consumer closeError = IO.Consumer.throwing(CloseError::new); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.flowOf(X::open, X::read, X::close); - assertThatCode(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - } - }).doesNotThrowAnyException(); - assertThat(c).containsExactly("open", "read", "close"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.flowOf(X::open, X::read, X::close); - assertThatThrownBy(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - throw new Error1(); - } - }).isInstanceOf(Error1.class); - assertThat(c).containsExactly("open", "read", "close"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.valueOf(openError, X::read, X::close); - assertThatThrownBy(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - } - }).isInstanceOf(OpenError.class); - assertThat(c).isEmpty(); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, readError, X::close); - assertThatThrownBy(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - } - }).isInstanceOf(ReadError.class); - assertThat(c).containsExactly("open", "close"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, X::read, closeError); - assertThatThrownBy(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - } - }).isInstanceOf(CloseError.class); - assertThat(c).containsExactly("open", "read"); - }); - - assertThat(new ArrayList()).satisfies(c -> { - IO.Parser, Closeable> p = IO.Parser.valueOf(X::open, readError, closeError); - assertThatThrownBy(() -> { - try (AutoCloseable auto = p.parseWithIO((List) c)) { - } - }).isInstanceOf(ReadError.class).hasSuppressedException(new CloseError()); - assertThat(c).containsExactly("open"); - }); - } - - private static final class OpenError extends IOException { - } - - private static final class ReadError extends IOException { - } - - private static final class CloseError extends IOException { - } - - private static final class Error1 extends IOException { - } - - private static final class Error2 extends IOException { - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java index 2e7594bb..3d9592c8 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/RestExecutorImpl.java @@ -17,7 +17,7 @@ package internal.web.sdmx21; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.util.IO; +import ioutil.IO; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java index 6c24e7e2..b5060828 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java @@ -27,10 +27,10 @@ import be.nbb.sdmx.facade.util.SdmxExceptions; import static be.nbb.sdmx.facade.util.SdmxMediaType.*; import static be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams.*; -import be.nbb.util.IO; import be.nbb.util.Stax; import internal.web.RestClient; import static internal.web.sdmx21.Sdmx21RestQueries.*; +import ioutil.IO; import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -56,7 +56,7 @@ public List getFlows() throws IOException { URL url = getFlowsQuery(endpoint); return flow21(langs) .onInputStream(Stax.getInputFactory(), UTF_8) - .parseWithIO(calling(url, XML)); + .applyWithIO(calling(url, XML)); } @Override @@ -64,7 +64,7 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { URL url = getFlowQuery(endpoint, ref); return flow21(langs) .onInputStream(Stax.getInputFactory(), UTF_8) - .parseWithIO(calling(url, XML)) + .applyWithIO(calling(url, XML)) .stream() .filter(ref::containsRef) .findFirst() @@ -76,7 +76,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { URL url = getStructureQuery(endpoint, ref); return struct21(langs) .onInputStream(Stax.getInputFactory(), UTF_8) - .parseWithIO(calling(url, STRUCTURE_21)) + .applyWithIO(calling(url, STRUCTURE_21)) .stream() .filter(ref::equalsRef) .findFirst() @@ -88,7 +88,7 @@ public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure ds URL url = getDataQuery(endpoint, flowRef, query); return compactData21(dsd, dialect) .onInputStream(Stax.getInputFactoryWithoutNamespace(), UTF_8) - .parseWithIO(calling(url, STRUCTURE_SPECIFIC_DATA_21)); + .applyWithIO(calling(url, STRUCTURE_SPECIFIC_DATA_21)); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java index e46fa348..9f2a634f 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeRestClientTest.java @@ -19,8 +19,8 @@ import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; -import be.nbb.util.IO; import be.nbb.sdmx.facade.util.UnexpectedIOException; +import ioutil.IO; import java.io.IOException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java index 47b98d13..de31b2d8 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/DriverAssertions.java @@ -18,10 +18,10 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.util.IO; import be.nbb.sdmx.facade.web.SdmxWebEntryPoint; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.web.RestDriverSupport; +import ioutil.IO; import static org.assertj.core.api.Assertions.*; /** diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java index 3c204c21..9ed7647f 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java @@ -95,7 +95,7 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference ECB = new AtomicReference<>(); private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.struct20(l).onReader(f).applyWithIO(xml::openReader); } private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { @@ -106,21 +106,21 @@ private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorit } List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).parseWithIO(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).applyWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.struct21(l).onReader(f).applyWithIO(xml::openReader); } private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).onReader(f).parseWithIO(xml::openReader); + return SdmxXmlStreams.flow21(l).onReader(f).applyWithIO(xml::openReader); } List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).parseWithIO(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).applyWithIO(xml::openReader)) { return SeriesSupport.copyOf(c); } } From 6de3737e28fb24adfd20c8131e7ef295a93044c2 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 7 Dec 2017 13:43:40 +0100 Subject: [PATCH 62/68] Added missing repositories. --- sdmx-facade/pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index b7e72332..57f7a393 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -110,4 +110,15 @@ + + + + oss-jfrog-artifactory-releases + https://oss.jfrog.org/artifactory/oss-release-local + + + oss-jfrog-artifactory-snapshots + https://oss.jfrog.org/artifactory/oss-snapshot-local + + \ No newline at end of file From 36b8cb2bc990c9ee00cf87be70468c3fba94a393 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 18 Dec 2017 16:42:55 +0100 Subject: [PATCH 63/68] Moved Stax util to its own project. --- .../java/test/samples/FacadeResource.java | 40 +++-- .../nbb/sdmx/facade/file/SdmxFileManager.java | 9 +- .../java/internal/file/CachedResource.java | 5 +- .../internal/file/SdmxDecoderResource.java | 10 +- .../main/java/internal/file/SdmxFileUtil.java | 4 +- .../file/xml/DataStructureDecoder.java | 32 ++-- .../java/internal/file/xml/DataTypeProbe.java | 10 +- .../internal/file/xml/StaxSdmxDecoder.java | 24 +-- .../file/SdmxFileConnectionImplTest.java | 9 +- .../file/xml/DataStructureDecoderTest.java | 12 +- .../internal/file/xml/DataTypeProbeTest.java | 13 +- .../facade/xml/stream/SdmxXmlStreams.java | 61 +++++--- .../stream/XMLStreamCompactDataCursor.java | 11 +- .../facade/xml/stream/XMLStreamFlow21.java | 4 +- .../stream/XMLStreamGenericDataCursor.java | 11 +- .../xml/stream/XMLStreamStructure20.java | 6 +- .../xml/stream/XMLStreamStructure21.java | 6 +- .../be/nbb/util/{Stax.java => StaxUtil.java} | 111 +------------- .../sdmx/facade/util/SeriesSupportTest.java | 11 +- .../XMLStreamCompactDataCursorTest.java | 4 +- .../xml/stream/XMLStreamFlow21Test.java | 10 +- .../XMLStreamGenericDataCursorTest.java | 4 +- .../xml/stream/XMLStreamStructure20Test.java | 15 +- .../xml/stream/XMLStreamStructure21Test.java | 11 +- .../src/test/java/be/nbb/util/StaxTest.java | 137 ------------------ .../internal/web/sdmx21/Sdmx21RestClient.java | 14 +- .../java/test/samples/ConnectorsResource.java | 7 +- .../java/test/samples/FacadeResource.java | 40 +++-- 28 files changed, 188 insertions(+), 443 deletions(-) rename sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/{Stax.java => StaxUtil.java} (64%) delete mode 100644 sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index 9ed7647f..3fcde583 100644 --- a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -28,12 +28,10 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.util.Stax; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import javax.xml.stream.XMLInputFactory; /** * @@ -52,9 +50,9 @@ public SdmxRepository nbb() throws IOException { if (result == null) { LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); + List structs = struct20(SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(SdmxSource.NBB_DATA, structs.get(0)); result = SdmxRepository.builder() .structures(structs) @@ -74,9 +72,9 @@ public SdmxRepository ecb() throws IOException { if (result == null) { LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); - List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); - List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); + List structs = struct21(SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(SdmxSource.ECB_DATAFLOWS, l); + List data = data21(SdmxSource.ECB_DATA, structs.get(0)); result = SdmxRepository.builder() .structures(structs) @@ -94,33 +92,33 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference NBB = new AtomicReference<>(); private static final AtomicReference ECB = new AtomicReference<>(); - private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).onReader(f).applyWithIO(xml::openReader); + private List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct20(l).parseReader(xml::openReader); } - private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + private List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { // FIXME: find sample of dataflow20 ? - return struct20(f, xml, l).stream() + return struct20(xml, l).stream() .map(FacadeResource::asDataflow) .collect(Collectors.toList()); } - List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).applyWithIO(xml::openReader)) { + List data20(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } - private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).onReader(f).applyWithIO(xml::openReader); + private List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct21(l).parseReader(xml::openReader); } - private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).onReader(f).applyWithIO(xml::openReader); + private List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.flow21(l).parseReader(xml::openReader); } - List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).applyWithIO(xml::openReader)) { + List data21(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } @@ -129,6 +127,4 @@ private Dataflow asDataflow(DataStructure o) { DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); return Dataflow.of(ref, o.getRef(), o.getLabel()); } - - private final XMLInputFactory XIF = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java index 10ccf06d..f4415eb0 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileManager.java @@ -23,7 +23,6 @@ import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.util.Stax; import internal.file.CachedResource; import internal.file.SdmxDecoder; import internal.file.SdmxFileConnectionImpl; @@ -37,7 +36,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import javax.annotation.Nonnull; -import javax.xml.stream.XMLInputFactory; import lombok.AccessLevel; /** @@ -51,10 +49,8 @@ public final class SdmxFileManager implements SdmxConnectionSupplier, HasCache { public static SdmxFileManager ofServiceLoader() { List dialects = new ArrayList<>(); ServiceLoader.load(SdmxDialect.class).forEach(dialects::add); - XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); return new SdmxFileManager( - factoryWithoutNamespace, - new StaxSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace), + new StaxSdmxDecoder(), HasCache.of(ConcurrentHashMap::new), dialects ); @@ -62,7 +58,6 @@ public static SdmxFileManager ofServiceLoader() { private static final DataStructureRef EMPTY = DataStructureRef.of("", "", ""); - private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final HasCache cacheSupport; private final List dialects; @@ -106,7 +101,7 @@ private SdmxFileSet getFiles(String name) throws IOException { } private SdmxFileConnectionImpl.Resource getResource(SdmxFileSet files, LanguagePriorityList languages) { - return new CachedResource(files, languages, factoryWithoutNamespace, decoder, getDataFactory(files), getCache()); + return new CachedResource(files, languages, decoder, getDataFactory(files), getCache()); } private Dataflow getDataflow(SdmxFileSet files) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java index 4079c94d..78ffdeff 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/CachedResource.java @@ -32,7 +32,6 @@ import java.util.Optional; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import javax.xml.stream.XMLInputFactory; /** * @@ -48,8 +47,8 @@ public final class CachedResource extends SdmxDecoderResource { private final TypedId decodeKey; private final TypedId> loadDataKey; - public CachedResource(SdmxFileSet files, LanguagePriorityList languages, XMLInputFactory factoryWithoutNamespace, SdmxDecoder decoder, Optional dataFactory, ConcurrentMap cache) { - super(files, languages, factoryWithoutNamespace, decoder, dataFactory); + public CachedResource(SdmxFileSet files, LanguagePriorityList languages, SdmxDecoder decoder, Optional dataFactory, ConcurrentMap cache) { + super(files, languages, decoder, dataFactory); this.cache = TtlCache.of(cache, CLOCK, DEFAULT_CACHE_TTL); String base = SdmxFileUtil.toXml(files) + languages.toString(); this.decodeKey = TypedId.of("decode://" + base); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java index 9f5d25f5..f680472e 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxDecoderResource.java @@ -24,11 +24,9 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.util.Stax; +import ioutil.Xml; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Optional; -import javax.xml.stream.XMLInputFactory; /** * @@ -39,7 +37,6 @@ class SdmxDecoderResource implements SdmxFileConnectionImpl.Resource { private final SdmxFileSet files; private final LanguagePriorityList languages; - private final XMLInputFactory factoryWithoutNamespace; private final SdmxDecoder decoder; private final Optional dataFactory; @@ -51,11 +48,10 @@ public SdmxDecoder.Info decode() throws IOException { @Override public DataCursor loadData(SdmxDecoder.Info entry, DataflowRef flowRef, Key key, boolean serieskeysonly) throws IOException { return getDataSupplier(entry.getType(), entry.getStructure()) - .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) - .applyWithIO(files.getData()); + .parseFile(files.getData()); } - private Stax.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { + private Xml.Parser getDataSupplier(SdmxDecoder.DataType o, DataStructure dsd) throws IOException { switch (o) { case GENERIC20: return SdmxXmlStreams.genericData20(dsd, dataFactory.orElse(DataFactory.sdmx20())); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java index 4a7ce44b..65dbbdde 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileUtil.java @@ -17,7 +17,7 @@ package internal.file; import be.nbb.sdmx.facade.file.SdmxFileSet; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import java.io.File; import java.io.StringReader; import java.io.StringWriter; @@ -73,7 +73,7 @@ public static SdmxFileSet fromXml(@Nonnull String input) throws IllegalArgumentE String structure = null; String dialect = null; try { - XMLStreamReader xml = Stax.getInputFactoryWithoutNamespace().createXMLStreamReader(new StringReader(input)); + XMLStreamReader xml = StaxUtil.getInputFactoryWithoutNamespace().createXMLStreamReader(new StringReader(input)); while (xml.hasNext()) { if (xml.next() == XMLStreamReader.START_ELEMENT && xml.getLocalName().equals(ROOT_TAG)) { data = xml.getAttributeValue(null, DATA_ATTR); diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java index 99f08989..1454667c 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataStructureDecoder.java @@ -19,11 +19,13 @@ import be.nbb.sdmx.facade.DataStructure; import static internal.file.SdmxDecoder.DataType.*; import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; +import be.nbb.util.StaxUtil; +import ioutil.Stax; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.util.Stax; +import ioutil.Xml; /** * @@ -31,20 +33,32 @@ */ final class DataStructureDecoder { - public static Stax.Parser generic20() { - return Stax.Parser.of(DataStructureDecoder::generic20); + public static Xml.Parser generic20() { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler(Stax.FlowHandler.of(DataStructureDecoder::generic20)) + .build(); } - public static Stax.Parser compact20() { - return Stax.Parser.of(DataStructureDecoder::compact20); + public static Xml.Parser compact20() { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler(Stax.FlowHandler.of(DataStructureDecoder::compact20)) + .build(); } - public static Stax.Parser generic21() { - return Stax.Parser.of(DataStructureDecoder::generic21); + public static Xml.Parser generic21() { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler(Stax.FlowHandler.of(DataStructureDecoder::generic21)) + .build(); } - public static Stax.Parser compact21() { - return Stax.Parser.of(DataStructureDecoder::compact21); + public static Xml.Parser compact21() { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler(Stax.FlowHandler.of(DataStructureDecoder::compact21)) + .build(); } private static boolean isTagMatch(XMLStreamReader r, String tag) { diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java index f114a075..e062e370 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java @@ -17,13 +17,15 @@ package internal.file.xml; import static be.nbb.sdmx.facade.xml.Sdmxml.*; +import be.nbb.util.StaxUtil; import static internal.file.SdmxDecoder.DataType.*; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import be.nbb.util.Stax; import internal.file.SdmxDecoder; +import ioutil.Stax; +import ioutil.Xml; /** * @@ -31,12 +33,12 @@ */ final class DataTypeProbe { - public static Stax.Parser of() { - return Stax.Parser.of(DataTypeProbe::probeDataType); + public static Xml.Parser of() { + return Stax.StreamParser.valueOf(DataTypeProbe::probeDataType); } private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws XMLStreamException { - if (Stax.isNotNamespaceAware(reader)) { + if (StaxUtil.isNotNamespaceAware(reader)) { throw new XMLStreamException("Cannot probe data type"); } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java index 81309c79..9eb9e56d 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/StaxSdmxDecoder.java @@ -20,8 +20,6 @@ import be.nbb.sdmx.facade.LanguagePriorityList; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import javax.xml.stream.XMLInputFactory; import static internal.file.SdmxDecoder.DataType.COMPACT20; import static internal.file.SdmxDecoder.DataType.COMPACT21; import static internal.file.SdmxDecoder.DataType.GENERIC20; @@ -29,9 +27,9 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.util.List; -import be.nbb.util.Stax; import internal.file.SdmxDecoder; import internal.file.SdmxFileUtil; +import ioutil.Xml; /** * @@ -40,9 +38,6 @@ @lombok.AllArgsConstructor public final class StaxSdmxDecoder implements SdmxDecoder { - private final XMLInputFactory factory; - private final XMLInputFactory factoryWithoutNamespace; - @Override public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOException { DataType type = probeDataType(files.getData()); @@ -53,19 +48,14 @@ public Info decode(SdmxFileSet files, LanguagePriorityList langs) throws IOExcep } private DataType probeDataType(File data) throws IOException { - return DataTypeProbe.of() - .onFile(factory, StandardCharsets.UTF_8) - .applyWithIO(data); + return DataTypeProbe.of().parseFile(data); } private DataStructure parseStruct(DataType dataType, LanguagePriorityList langs, File structure) throws IOException { - return getStructParser(dataType, langs) - .onFile(factory, StandardCharsets.UTF_8) - .applyWithIO(structure) - .get(0); + return getStructParser(dataType, langs).parseFile(structure).get(0); } - private Stax.Parser> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { + private Xml.Parser> getStructParser(DataType o, LanguagePriorityList langs) throws IOException { switch (o) { case GENERIC20: case COMPACT20: @@ -79,12 +69,10 @@ private Stax.Parser> getStructParser(DataType o, LanguagePri } private DataStructure decodeStruct(DataType dataType, File data) throws IOException { - return getStructDecoder(dataType) - .onFile(factoryWithoutNamespace, StandardCharsets.UTF_8) - .applyWithIO(data); + return getStructDecoder(dataType).parseFile(data); } - private static Stax.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { + private static Xml.Parser getStructDecoder(SdmxDecoder.DataType o) throws IOException { switch (o) { case GENERIC20: return DataStructureDecoder.generic20(); diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index ba4e03c5..946b50d4 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -29,12 +29,10 @@ import be.nbb.sdmx.facade.file.SdmxFileSet; import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.ConnectionAssert; -import be.nbb.util.Stax; import java.io.File; import java.io.IOException; import java.util.Optional; import java.util.stream.Stream; -import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.*; import org.junit.Test; import org.junit.Rule; @@ -54,7 +52,7 @@ public void testFile() throws IOException { SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, decoder, Optional.empty()); SdmxFileConnectionImpl conn = new SdmxFileConnectionImpl(r, dataflow); assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); @@ -74,7 +72,7 @@ public void testCompactData21() throws IOException { SdmxFileSet files = SdmxFileSet.builder().data(compact21).build(); - SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, factoryWithoutNamespace, decoder, Optional.empty()); + SdmxFileConnectionImpl.Resource r = new SdmxDecoderResource(files, ANY, decoder, Optional.empty()); SdmxFileConnectionImpl conn = new SdmxFileConnectionImpl(r, dataflow); assertThat(conn.getFlows()).hasSize(1); @@ -109,7 +107,6 @@ public void testCompactData21() throws IOException { @Rule public TemporaryFolder temp = new TemporaryFolder(); - private final XMLInputFactory factoryWithoutNamespace = Stax.getInputFactoryWithoutNamespace(); - private final SdmxDecoder decoder = new StaxSdmxDecoder(Stax.getInputFactory(), factoryWithoutNamespace); + private final SdmxDecoder decoder = new StaxSdmxDecoder(); private final Dataflow dataflow = Dataflow.of(DataflowRef.parse("data"), DataStructureRef.parse("xyz"), "label"); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java index c2fdc0bc..f11f134f 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataStructureDecoderTest.java @@ -21,9 +21,7 @@ import java.io.IOException; import org.junit.Test; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.util.Stax; import static org.assertj.core.api.Assertions.assertThat; -import javax.xml.stream.XMLInputFactory; import static internal.file.xml.CustomDataStructureBuilder.dimension; /** @@ -45,7 +43,7 @@ public void testDecodeGeneric20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic20().onReader(xif).applyWithIO(SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic20().parseReader(SdmxSource.OTHER_GENERIC20::openReader)).isEqualTo(ds); } @Test @@ -62,7 +60,7 @@ public void testDecodeCompact20() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact20().onReader(xif).applyWithIO(SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact20().parseReader(SdmxSource.OTHER_COMPACT20::openReader)).isEqualTo(ds); } @Test @@ -81,7 +79,7 @@ public void testDecodeGeneric21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.generic21().onReader(xif).applyWithIO(SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.generic21().parseReader(SdmxSource.OTHER_GENERIC21::openReader)).isEqualTo(ds); } @Test @@ -100,8 +98,6 @@ public void testDecodeCompact21() throws IOException { .primaryMeasureId("OBS_VALUE") .build(); - assertThat(DataStructureDecoder.compact21().onReader(xif).applyWithIO(SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); + assertThat(DataStructureDecoder.compact21().parseReader(SdmxSource.OTHER_COMPACT21::openReader)).isEqualTo(ds); } - - private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); } diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java index 856474a9..1f75ccc4 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/xml/DataTypeProbeTest.java @@ -17,11 +17,8 @@ package internal.file.xml; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.util.Stax; import internal.file.SdmxDecoder; -import ioutil.IO; import java.io.IOException; -import java.io.Reader; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -33,27 +30,25 @@ public class DataTypeProbeTest { @Test public void testDecodeGeneric20() throws IOException { - assertThat(decoder.applyWithIO(SdmxSource.OTHER_GENERIC20::openReader)) + assertThat(DataTypeProbe.of().parseReader(SdmxSource.OTHER_GENERIC20::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC20); } @Test public void testDecodeCompact20() throws IOException { - assertThat(decoder.applyWithIO(SdmxSource.OTHER_COMPACT20::openReader)) + assertThat(DataTypeProbe.of().parseReader(SdmxSource.OTHER_COMPACT20::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT20); } @Test public void testDecodeGeneric21() throws IOException { - assertThat(decoder.applyWithIO(SdmxSource.OTHER_GENERIC21::openReader)) + assertThat(DataTypeProbe.of().parseReader(SdmxSource.OTHER_GENERIC21::openReader)) .isEqualTo(SdmxDecoder.DataType.GENERIC21); } @Test public void testDecodeCompact21() throws IOException { - assertThat(decoder.applyWithIO(SdmxSource.OTHER_COMPACT21::openReader)) + assertThat(DataTypeProbe.of().parseReader(SdmxSource.OTHER_COMPACT21::openReader)) .isEqualTo(SdmxDecoder.DataType.COMPACT21); } - - private final IO.Function, SdmxDecoder.DataType> decoder = DataTypeProbe.of().onReader(Stax.getInputFactory()); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index 7da6c466..ce57cefb 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -16,7 +16,6 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataStructure; @@ -24,6 +23,9 @@ import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.parser.ObsParser; +import be.nbb.util.StaxUtil; +import ioutil.Stax; +import ioutil.Xml; import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; @@ -36,57 +38,78 @@ public class SdmxXmlStreams { @Nonnull - public Stax.Parser compactData20(@Nonnull DataStructure dsd) throws IOException { + public Stax.StreamParser compactData20(@Nonnull DataStructure dsd) throws IOException { return compactData20(dsd, DataFactory.sdmx20()); } @Nonnull - public Stax.Parser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + public Stax.StreamParser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler((o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId())) + .build(); } @Nonnull - public Stax.Parser compactData21(@Nonnull DataStructure dsd) throws IOException { + public Stax.StreamParser compactData21(@Nonnull DataStructure dsd) throws IOException { return compactData21(dsd, DataFactory.sdmx21()); } @Nonnull - public Stax.Parser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId()); + public Stax.StreamParser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler((o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId())) + .build(); } @Nonnull - public Stax.Parser genericData20(@Nonnull DataStructure dsd) throws IOException { + public Stax.StreamParser genericData20(@Nonnull DataStructure dsd) throws IOException { return genericData20(dsd, DataFactory.sdmx20()); } @Nonnull - public Stax.Parser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd)); + public Stax.StreamParser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler((o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd))) + .build(); } @Nonnull - public Stax.Parser genericData21(@Nonnull DataStructure dsd) throws IOException { + public Stax.StreamParser genericData21(@Nonnull DataStructure dsd) throws IOException { return genericData21(dsd, DataFactory.sdmx21()); } @Nonnull - public Stax.Parser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { - return (o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd)); + public Stax.StreamParser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + return Stax.StreamParser.builder() + .factory(StaxUtil::getInputFactoryWithoutNamespace) + .handler((o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd))) + .build(); } @Nonnull - public Stax.Parser> struct20(@Nonnull LanguagePriorityList langs) throws IOException { - return Stax.Parser.of(new XMLStreamStructure20(langs)::parse); + public Xml.Parser> struct20(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.StreamParser.>builder() + .factory(StaxUtil::getInputFactory) + .handler(Stax.FlowHandler.of(new XMLStreamStructure20(langs)::parse)) + .build(); } @Nonnull - public Stax.Parser> struct21(@Nonnull LanguagePriorityList langs) throws IOException { - return Stax.Parser.of(new XMLStreamStructure21(langs)::parse); + public Xml.Parser> struct21(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.StreamParser.>builder() + .factory(StaxUtil::getInputFactory) + .handler(Stax.FlowHandler.of(new XMLStreamStructure21(langs)::parse)) + .build(); } @Nonnull - public Stax.Parser> flow21(@Nonnull LanguagePriorityList langs) throws IOException { - return Stax.Parser.of(new XMLStreamFlow21(langs)::parse); + public Xml.Parser> flow21(@Nonnull LanguagePriorityList langs) throws IOException { + return Stax.StreamParser.>builder() + .factory(StaxUtil::getInputFactory) + .handler(Stax.FlowHandler.of(new XMLStreamFlow21(langs)::parse)) + .build(); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java index d646a30d..f2c0dbe3 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; @@ -32,6 +32,7 @@ import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.parser.Freqs; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; +import ioutil.Xml; import java.io.Closeable; /** @@ -58,7 +59,7 @@ final class XMLStreamCompactDataCursor implements DataCursor { private boolean hasObs; XMLStreamCompactDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser, String timeDimensionId, String primaryMeasureId) { - if (!Stax.isNotNamespaceAware(reader)) { + if (!StaxUtil.isNotNamespaceAware(reader)) { log.fine("Using XMLStreamReader with namespace awareness"); } this.reader = reader; @@ -82,7 +83,7 @@ public boolean nextSeries() throws IOException { try { return hasSeries = nextWhile(this::onDataSet); } catch (XMLStreamException ex) { - throw new Stax.XMLStreamIOException(ex); + throw new Xml.WrappedException(ex); } } @@ -93,7 +94,7 @@ public boolean nextObs() throws IOException { try { return hasObs = nextWhile(this::onSeriesBody); } catch (XMLStreamException ex) { - throw new Stax.XMLStreamIOException(ex); + throw new Xml.WrappedException(ex); } } @@ -139,7 +140,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - Stax.closeBoth(reader, onClose); + StaxUtil.closeBoth(reader, onClose); } private void checkState() throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java index bb63f348..d8aada5a 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; @@ -58,7 +58,7 @@ final class XMLStreamFlow21 { @Nonnull public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { - if (Stax.isNotNamespaceAware(reader)) { + if (StaxUtil.isNotNamespaceAware(reader)) { throw new XMLStreamException("Cannot parse flows"); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java index 5b842147..49aadab8 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; @@ -32,6 +32,7 @@ import javax.xml.stream.XMLStreamReader; import be.nbb.sdmx.facade.parser.Freqs; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.isTagMatch; +import ioutil.Xml; import java.io.Closeable; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -73,7 +74,7 @@ static XMLStreamGenericDataCursor sdmx21(XMLStreamReader reader, Closeable onClo private boolean hasObs; private XMLStreamGenericDataCursor(XMLStreamReader reader, Closeable onClose, Key.Builder keyBuilder, ObsParser obsParser, Freqs.Parser freqParser, SeriesHeadParser headParser) { - if (!Stax.isNotNamespaceAware(reader)) { + if (!StaxUtil.isNotNamespaceAware(reader)) { log.fine("Using XMLStreamReader with namespace awareness"); } this.reader = reader; @@ -96,7 +97,7 @@ public boolean nextSeries() throws IOException { try { return hasSeries = nextWhile(this::onDataSet); } catch (XMLStreamException ex) { - throw new Stax.XMLStreamIOException(ex); + throw new Xml.WrappedException(ex); } } @@ -114,7 +115,7 @@ public boolean nextObs() throws IOException { } return hasObs = nextWhile(this::onSeriesBody); } catch (XMLStreamException ex) { - throw new Stax.XMLStreamIOException(ex); + throw new Xml.WrappedException(ex); } } @@ -160,7 +161,7 @@ public Double getObsValue() throws IOException { @Override public void close() throws IOException { closed = true; - Stax.closeBoth(reader, onClose); + StaxUtil.closeBoth(reader, onClose); } private void checkState() throws IOException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index cd9db047..3533337d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; @@ -35,6 +35,8 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * @@ -76,7 +78,7 @@ final class XMLStreamStructure20 { @Nonnull public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { - if (Stax.isNotNamespaceAware(reader)) { + if (StaxUtil.isNotNamespaceAware(reader)) { throw new XMLStreamException("Cannot parse structure"); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 97efc236..17f7e802 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; @@ -35,6 +35,8 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * @@ -78,7 +80,7 @@ final class XMLStreamStructure21 { @Nonnull public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamException { - if (Stax.isNotNamespaceAware(reader)) { + if (StaxUtil.isNotNamespaceAware(reader)) { throw new XMLStreamException("Cannot parse structure"); } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/StaxUtil.java similarity index 64% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java rename to sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/StaxUtil.java index ba069270..e9dbdc2c 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/Stax.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/StaxUtil.java @@ -19,15 +19,9 @@ import ioutil.IO; import ioutil.Xml; import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; import javax.annotation.Nonnull; import javax.xml.stream.EventFilter; import javax.xml.stream.StreamFilter; @@ -45,115 +39,18 @@ * @author Philippe Charles */ @lombok.experimental.UtilityClass -public class Stax { - - @FunctionalInterface - public interface Supplier { - - T getWithStream() throws XMLStreamException; - } - - @FunctionalInterface - public interface Function { - - R applyWithStream(T t) throws XMLStreamException; - } - - @FunctionalInterface - public interface Parser { - - @Nonnull - T parse(@Nonnull XMLStreamReader reader, @Nonnull Closeable onClose) throws IOException; - - @Nonnull - default IO.Function onFile(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { - return source -> parseStream(this, xf, () -> new FileInputStream(source), cs); - } - - @Nonnull - default IO.Function onPath(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { - return source -> { - Optional file = IO.getFile(source); - return file.isPresent() - ? parseStream(this, xf, () -> new FileInputStream(file.get()), cs) - : parseReader(this, xf, () -> Files.newBufferedReader(source, cs)); - }; - } - - @Nonnull - default IO.Function, T> onInputStream(@Nonnull XMLInputFactory xf, @Nonnull Charset cs) { - return source -> parseStream(this, xf, source, cs); - } - - @Nonnull - default IO.Function, T> onReader(@Nonnull XMLInputFactory xf) { - return source -> parseReader(this, xf, source); - } - - @Nonnull - static Parser of(@Nonnull Function func) { - return (reader, onClose) -> { - try (Closeable c = onClose) { - return func.applyWithStream(reader); - } catch (XMLStreamException ex) { - throw new XMLStreamIOException(ex); - } - }; - } - } - - @Nonnull - private T parse(Parser parser, @Nonnull Supplier supplier, @Nonnull Closeable onClose) throws IOException { - try { - XMLStreamReader xml = supplier.getWithStream(); - return parser.parse(xml, () -> closeBoth(xml, onClose)); - } catch (XMLStreamException ex) { - ensureClosed(ex, onClose); - throw new XMLStreamIOException("Failed to create XMLStreamReader", ex); - } - } - - @Nonnull - T parseStream(Parser parser, @Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source, @Nonnull Charset cs) throws IOException { - InputStream stream = source.getWithIO(); - return parse(parser, () -> xf.createXMLStreamReader(stream, cs.name()), stream); - } - - @Nonnull - T parseReader(Parser parser, @Nonnull XMLInputFactory xf, @Nonnull IO.Supplier source) throws IOException { - Reader reader = source.getWithIO(); - return parse(parser, () -> xf.createXMLStreamReader(reader), reader); - } - - public static final class XMLStreamIOException extends IOException { - - public XMLStreamIOException(XMLStreamException ex) { - super(ex); - } - - public XMLStreamIOException(String message, XMLStreamException ex) { - super(message, ex); - } - } +public class StaxUtil { public void closeBoth(XMLStreamReader reader, Closeable onClose) throws IOException { try { reader.close(); } catch (XMLStreamException ex) { - ensureClosed(ex, onClose); - throw new Stax.XMLStreamIOException("Failed to close xml stream reader", ex); + IO.ensureClosed(ex, onClose); + throw new Xml.WrappedException(ex); } onClose.close(); } - void ensureClosed(XMLStreamException ex, Closeable onClose) throws IOException { - try { - onClose.close(); - } catch (IOException suppressed) { - ex.addSuppressed(suppressed); - } - } - @Nonnull public XMLInputFactory getInputFactory() { return ImmutableInputFactory.DEFAULT; @@ -180,7 +77,7 @@ private ImmutableInputFactory(boolean namespaceAware) { if (!namespaceAware && delegate.isPropertySupported(XMLInputFactory.IS_NAMESPACE_AWARE)) { delegate.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, false); } - Xml.StAX.preventXXE(delegate); + ioutil.Stax.preventXXE(delegate); } @Override diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 10cbb5e6..07507159 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -26,12 +26,10 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.util.Stax; import java.io.IOException; import java.time.LocalDateTime; import java.util.Collections; import java.util.List; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -76,8 +74,8 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); } - List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).applyWithIO(SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOf(c)).hasSize(120); } } @@ -99,8 +97,8 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(ANY).onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).onReader(xif).applyWithIO(SdmxSource.ECB_DATA::openReader)) { + List dsds = SdmxXmlStreams.struct21(ANY).parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader); + try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.ECB_DATA::openReader)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(120); @@ -108,5 +106,4 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { } private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); - private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index cc075913..9ff9ac32 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; @@ -121,5 +121,5 @@ public void testCompactData21() throws Exception { } } - private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); + private final XMLInputFactory xif = StaxUtil.getInputFactoryWithoutNamespace(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java index bd8d82ca..5dd511ff 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java @@ -21,10 +21,10 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; -import be.nbb.util.Stax; +import ioutil.Stax; +import ioutil.Xml; import java.io.IOException; import java.util.List; -import javax.xml.stream.XMLInputFactory; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -36,15 +36,13 @@ public class XMLStreamFlow21Test { @Test public void test() throws IOException { - Stax.Parser> p = Stax.Parser.of(new XMLStreamFlow21(LanguagePriorityList.ANY)::parse); + Xml.Parser> p = Stax.StreamParser.valueOf(new XMLStreamFlow21(LanguagePriorityList.ANY)::parse); - assertThat(p.onReader(xif).applyWithIO(SdmxSource.ECB_DATAFLOWS::openReader)) + assertThat(p.parseReader(SdmxSource.ECB_DATAFLOWS::openReader)) .containsExactly( Dataflow.of(DataflowRef.of("ECB", "AME", "1.0"), DataStructureRef.of("ECB", "ECB_AME1", "1.0"), "AMECO"), Dataflow.of(DataflowRef.of("ECB", "BKN", "1.0"), DataStructureRef.of("ECB", "ECB_BKN1", "1.0"), "Banknotes statistics"), Dataflow.of(DataflowRef.of("ECB", "BLS", "1.0"), DataStructureRef.of("ECB", "ECB_BLS1", "1.0"), "Bank Lending Survey Statistics") ); } - - private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 37383f8b..88753dbd 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; +import be.nbb.util.StaxUtil; import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; @@ -169,5 +169,5 @@ public void testGenericData21Bis() throws Exception { } } - private final XMLInputFactory xif = Stax.getInputFactoryWithoutNamespace(); + private final XMLInputFactory xif = StaxUtil.getInputFactoryWithoutNamespace(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java index 357b06e0..a16d0405 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java @@ -16,13 +16,12 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; +import ioutil.Xml; import java.util.List; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -36,9 +35,9 @@ public class XMLStreamStructure20Test { @Test @SuppressWarnings("null") public void test() throws Exception { - Stax.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); + Xml.Parser> p1 = SdmxXmlStreams.struct20(LanguagePriorityList.ANY); - assertThat(p1.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p1.parseReader(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("My first dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -50,9 +49,9 @@ public void test() throws Exception { }); }); - Stax.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); + Xml.Parser> p2 = SdmxXmlStreams.struct20(LanguagePriorityList.parse("fr")); - assertThat(p2.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(p2.parseReader(SdmxSource.NBB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("Mon premier dataset"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME"); @@ -65,10 +64,8 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> p1.onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> p1.parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } - - private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java index ba795cca..66830a92 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java @@ -16,13 +16,12 @@ */ package be.nbb.sdmx.facade.xml.stream; -import be.nbb.util.Stax; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.samples.SdmxSource; +import ioutil.Xml; import java.util.List; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -36,9 +35,9 @@ public class XMLStreamStructure21Test { @Test @SuppressWarnings("null") public void test() throws Exception { - Stax.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); + Xml.Parser> parser = SdmxXmlStreams.struct21(LanguagePriorityList.ANY); - assertThat(parser.onReader(xif).applyWithIO(SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { + assertThat(parser.parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader)).hasSize(1).element(0).satisfies(o -> { assertThat(o.getLabel()).isEqualTo("AMECO"); assertThat(o.getPrimaryMeasureId()).isEqualTo("OBS_VALUE"); assertThat(o.getTimeDimensionId()).isEqualTo("TIME_PERIOD"); @@ -51,10 +50,8 @@ public void test() throws Exception { }); assertThatIOException() - .isThrownBy(() -> parser.onReader(xif).applyWithIO(SdmxSource.NBB_DATA_STRUCTURE::openReader)) + .isThrownBy(() -> parser.parseReader(SdmxSource.NBB_DATA_STRUCTURE::openReader)) .withCauseInstanceOf(XMLStreamException.class) .withMessageContaining("Invalid namespace"); } - - private final XMLInputFactory xif = Stax.getInputFactory(); } diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java deleted file mode 100644 index 94a4d367..00000000 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/util/StaxTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.util; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import static java.nio.charset.StandardCharsets.UTF_8; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import static org.assertj.core.api.Assertions.*; -import org.junit.Test; - -/** - * - * @author Philippe Charles - */ -public class StaxTest { - - @Test - @SuppressWarnings("null") - public void testValidParseStream() throws IOException { - Stax.Parser p = (o, c) -> { - c.close(); - return ""; - }; - - assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, null, EmptyStream::new, UTF_8)); - - assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, xif, null, UTF_8)); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, OpenErrorStream::new, UTF_8)) - .isInstanceOf(OpenError.class) - .hasNoSuppressedExceptions() - .hasNoCause(); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, ReadErrorStream::new, UTF_8)) - .isInstanceOf(Stax.XMLStreamIOException.class) - .hasNoSuppressedExceptions() - .hasCauseInstanceOf(XMLStreamException.class) - .hasRootCauseInstanceOf(ReadError.class); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, CloseErrorStream::new, UTF_8)) - .isInstanceOf(CloseError.class) - .hasNoSuppressedExceptions() - .hasNoCause(); - } - - @Test - @SuppressWarnings("null") - public void testInvalidParseStream() throws IOException { - Stax.Parser p = (o, c) -> { - try (Closeable x = c) { - throw new ParseError(); - } - }; - - assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, null, EmptyStream::new, UTF_8)); - - assertThatNullPointerException().isThrownBy(() -> Stax.parseStream(p, xif, null, UTF_8)); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, OpenErrorStream::new, UTF_8)) - .isInstanceOf(OpenError.class) - .hasNoSuppressedExceptions() - .hasNoCause(); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, ReadErrorStream::new, UTF_8)) - .isInstanceOf(Stax.XMLStreamIOException.class) - .hasNoSuppressedExceptions() - .hasCauseInstanceOf(XMLStreamException.class) - .hasRootCauseInstanceOf(ReadError.class); - - assertThatThrownBy(() -> Stax.parseStream(p, xif, CloseErrorStream::new, UTF_8)) - .isInstanceOf(ParseError.class) - .hasSuppressedException(new CloseError()) - .hasNoCause(); - } - - private final XMLInputFactory xif = Stax.getInputFactory(); - - private static class EmptyStream extends InputStream { - - @Override - public int read() throws IOException { - return -1; - } - } - - private static final class OpenErrorStream extends EmptyStream { - - public OpenErrorStream() throws IOException { - throw new OpenError(); - } - } - - private static final class ReadErrorStream extends EmptyStream { - - @Override - public int read() throws IOException { - throw new ReadError(); - } - } - - private static final class CloseErrorStream extends EmptyStream { - - @Override - public void close() throws IOException { - throw new CloseError(); - } - } - - private static final class ParseError extends IOException { - } - - private static final class OpenError extends IOException { - } - - private static final class ReadError extends IOException { - } - - private static final class CloseError extends IOException { - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java index b5060828..56ac0c71 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/sdmx21/Sdmx21RestClient.java @@ -27,14 +27,12 @@ import be.nbb.sdmx.facade.util.SdmxExceptions; import static be.nbb.sdmx.facade.util.SdmxMediaType.*; import static be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams.*; -import be.nbb.util.Stax; import internal.web.RestClient; import static internal.web.sdmx21.Sdmx21RestQueries.*; import ioutil.IO; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import static java.nio.charset.StandardCharsets.UTF_8; import java.util.List; /** @@ -55,16 +53,14 @@ public final class Sdmx21RestClient implements RestClient { public List getFlows() throws IOException { URL url = getFlowsQuery(endpoint); return flow21(langs) - .onInputStream(Stax.getInputFactory(), UTF_8) - .applyWithIO(calling(url, XML)); + .parseStream(calling(url, XML)); } @Override public Dataflow getFlow(DataflowRef ref) throws IOException { URL url = getFlowQuery(endpoint, ref); return flow21(langs) - .onInputStream(Stax.getInputFactory(), UTF_8) - .applyWithIO(calling(url, XML)) + .parseStream(calling(url, XML)) .stream() .filter(ref::containsRef) .findFirst() @@ -75,8 +71,7 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { public DataStructure getStructure(DataStructureRef ref) throws IOException { URL url = getStructureQuery(endpoint, ref); return struct21(langs) - .onInputStream(Stax.getInputFactory(), UTF_8) - .applyWithIO(calling(url, STRUCTURE_21)) + .parseStream(calling(url, STRUCTURE_21)) .stream() .filter(ref::equalsRef) .findFirst() @@ -87,8 +82,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { URL url = getDataQuery(endpoint, flowRef, query); return compactData21(dsd, dialect) - .onInputStream(Stax.getInputFactoryWithoutNamespace(), UTF_8) - .applyWithIO(calling(url, STRUCTURE_SPECIFIC_DATA_21)); + .parseStream(calling(url, STRUCTURE_SPECIFIC_DATA_21)); } @Override diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java index 1312a311..0f77f4a4 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/ConnectorsResource.java @@ -24,7 +24,6 @@ import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.ObsParser; -import be.nbb.util.Stax; import internal.connectors.PortableTimeSeriesCursor; import internal.connectors.Util; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; @@ -106,7 +105,7 @@ private List flow20(ByteSource xml, LanguagePriorityList l) throws IOE private List data20(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data20(XIF, xml, Util.toStructure(dsd)) + return FacadeResource.data20(xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -122,7 +121,7 @@ private List flow21(ByteSource xml, LanguagePriorityList l) throws IOE public List data21(ByteSource xml, DataFlowStructure dsd, LanguagePriorityList l) throws IOException { // No connectors impl - return FacadeResource.data21(XIF, xml, Util.toStructure(dsd)) + return FacadeResource.data21(xml, Util.toStructure(dsd)) .stream() .map((Series o) -> toPortableTimeSeries(o, dsd.getDimensions())) .collect(Collectors.toList()); @@ -211,5 +210,5 @@ private char formatByStandardFreq(Frequency code) { } } - private final XMLInputFactory XIF = Stax.getInputFactory(); + private final XMLInputFactory XIF = XMLInputFactory.newFactory(); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java index 9ed7647f..3fcde583 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-web/src/test/java/test/samples/FacadeResource.java @@ -28,12 +28,10 @@ import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; -import be.nbb.util.Stax; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import javax.xml.stream.XMLInputFactory; /** * @@ -52,9 +50,9 @@ public SdmxRepository nbb() throws IOException { if (result == null) { LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List flows = flow20(XIF, SdmxSource.NBB_DATA_STRUCTURE, l); - List data = data20(XIF, SdmxSource.NBB_DATA, structs.get(0)); + List structs = struct20(SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(SdmxSource.NBB_DATA, structs.get(0)); result = SdmxRepository.builder() .structures(structs) @@ -74,9 +72,9 @@ public SdmxRepository ecb() throws IOException { if (result == null) { LanguagePriorityList l = LanguagePriorityList.parse("fr"); - List structs = struct21(XIF, SdmxSource.ECB_DATA_STRUCTURE, l); - List flows = flow21(XIF, SdmxSource.ECB_DATAFLOWS, l); - List data = data21(XIF, SdmxSource.ECB_DATA, structs.get(0)); + List structs = struct21(SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(SdmxSource.ECB_DATAFLOWS, l); + List data = data21(SdmxSource.ECB_DATA, structs.get(0)); result = SdmxRepository.builder() .structures(structs) @@ -94,33 +92,33 @@ public SdmxRepository ecb() throws IOException { private static final AtomicReference NBB = new AtomicReference<>(); private static final AtomicReference ECB = new AtomicReference<>(); - private List struct20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct20(l).onReader(f).applyWithIO(xml::openReader); + private List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct20(l).parseReader(xml::openReader); } - private List flow20(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { + private List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { // FIXME: find sample of dataflow20 ? - return struct20(f, xml, l).stream() + return struct20(xml, l).stream() .map(FacadeResource::asDataflow) .collect(Collectors.toList()); } - List data20(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).onReader(f).applyWithIO(xml::openReader)) { + List data20(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } - private List struct21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.struct21(l).onReader(f).applyWithIO(xml::openReader); + private List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct21(l).parseReader(xml::openReader); } - private List flow21(XMLInputFactory f, ByteSource xml, LanguagePriorityList l) throws IOException { - return SdmxXmlStreams.flow21(l).onReader(f).applyWithIO(xml::openReader); + private List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.flow21(l).parseReader(xml::openReader); } - List data21(XMLInputFactory f, ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).onReader(f).applyWithIO(xml::openReader)) { + List data21(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } @@ -129,6 +127,4 @@ private Dataflow asDataflow(DataStructure o) { DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); return Dataflow.of(ref, o.getRef(), o.getLabel()); } - - private final XMLInputFactory XIF = Stax.getInputFactory(); } From 45f3c0c2ad8e5d3db614b0ae13231ee67b60a5a2 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 20 Dec 2017 13:08:18 +0100 Subject: [PATCH 64/68] Fixed IMF endpoint. Fixed NPE when converting to connectors model. --- .../java/internal/connectors/ConnectorRestClient.java | 2 +- .../src/main/java/internal/connectors/Util.java | 9 +++++---- .../main/java/internal/connectors/drivers/ImfDriver.java | 6 +++--- .../java/internal/connectors/drivers/InseeDriver.java | 2 +- .../java/internal/connectors/drivers/Sdmx21Driver.java | 2 +- .../src/main/java/internal/web/FailsafeRestClient.java | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java index 4a9c559a..06fc3358 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java @@ -147,7 +147,7 @@ private static DataCursor getCursor(HasDataCursor connector, DataflowRef flowRef } private static DataCursor getAdaptedCursor(RestSdmxClient connector, DataflowRef flowRef, DataStructure dsd, DataQuery query) throws SdmxException { - return new PortableTimeSeriesCursor(connector.getTimeSeries(Util.fromFlowQuery(flowRef), Util.fromStructure(dsd), query.getKey().toString(), null, null, isSeriesKeyOnly(query), null, false), ObsParser.standard()); + return new PortableTimeSeriesCursor(connector.getTimeSeries(Util.fromFlowQuery(flowRef, dsd.getRef()), Util.fromStructure(dsd), query.getKey().toString(), null, null, isSeriesKeyOnly(query), null, false), ObsParser.standard()); } private static boolean isSeriesKeyOnly(DataQuery query) { diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java index 7b2141f7..1e02209e 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Util.java @@ -65,11 +65,12 @@ public DataStructure toStructure(DataFlowStructure dsd) { return result.build(); } - public Dataflow fromFlowQuery(be.nbb.sdmx.facade.DataflowRef ref) { + public Dataflow fromFlowQuery(be.nbb.sdmx.facade.DataflowRef flowRef, be.nbb.sdmx.facade.DataStructureRef structRef) { Dataflow result = new Dataflow(); - result.setAgency(ref.getAgency()); - result.setId(ref.getId()); - result.setVersion(ref.getVersion()); + result.setAgency(flowRef.getAgency()); + result.setId(flowRef.getId()); + result.setVersion(flowRef.getVersion()); + result.setDsdIdentifier(fromStructureRef(structRef)); return result; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java index e918a9b4..2e2d4cf8 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java @@ -17,11 +17,11 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.util.HasCache; -import it.bancaditalia.oss.sdmx.client.custom.IMF; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; import internal.web.RestDriverSupport; +import it.bancaditalia.oss.sdmx.client.custom.IMF2; /** * @@ -34,7 +34,7 @@ public final class ImfDriver implements SdmxWebDriver, HasCache { private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:imf:") - .client(ConnectorRestClient.of(IMF::new)) - .entry("IMF", "International Monetary Fund", "http://sdmxws.imf.org/SDMXRest/sdmx.ashx") + .client(ConnectorRestClient.of(IMF2::new)) + .entry("IMF", "International Monetary Fund", "http://dataservices.imf.org/REST/SDMX_XML.svc") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java index 3485b5b7..e7012d1c 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -136,7 +136,7 @@ private void loadMissingCodes(Codelist codelist) throws SdmxException { private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { return runQuery( getCompactData21Parser(dsd), - buildDataQuery(Util.fromFlowQuery(flowRef), resource.toString(), null, null, serieskeysonly, null, false), + buildDataQuery(Util.fromFlowQuery(flowRef, dsd.getRef()), resource.toString(), null, null, serieskeysonly, null, false), SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 2c7b43a4..4cdd951d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -206,7 +206,7 @@ public boolean isSeriesKeysOnlySupported() { private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { return runQuery( getCompactData21Parser(dsd), - buildDataQuery(Util.fromFlowQuery(flowRef), resource.toString(), null, null, serieskeysonly, null, false), + buildDataQuery(Util.fromFlowQuery(flowRef, dsd.getRef()), resource.toString(), null, null, serieskeysonly, null, false), SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java index 596cee47..c358358a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeRestClient.java @@ -125,7 +125,7 @@ public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException } private static IOException unexpected(RuntimeException ex, String format, Object... args) { - log.log(Level.WARNING, format, args); + log.log(Level.WARNING, String.format(format, args)); return new UnexpectedIOException(ex); } From 3d6e2c37b5127f41abf0f89431975041a97bf97e Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Wed, 20 Dec 2017 16:38:50 +0100 Subject: [PATCH 65/68] Fixed ILO driver. --- .../be/nbb/sdmx/facade/DataStructureRef.java | 10 ++++ .../connectors/drivers/IloDriver.java | 58 ++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java index ec78578c..1150531f 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructureRef.java @@ -39,6 +39,16 @@ public final class DataStructureRef implements ResourceRef { @lombok.NonNull private String version; + public boolean containsRef(@Nonnull DataStructure that) { + return contains(that.getRef()); + } + + public boolean contains(@Nonnull DataStructureRef that) { + return (this.agency.equals(ALL_AGENCIES) || this.agency.equals(that.agency)) + && (this.id.equals(that.id)) + && (this.version.equals(LATEST_VERSION) || this.version.equals(that.version)); + } + public boolean equalsRef(@Nonnull DataStructure that) { return equals(that.getRef()); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java index 5fa7c287..41ca725d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java @@ -17,11 +17,24 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.util.SdmxFix; import it.bancaditalia.oss.sdmx.client.custom.ILO; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; +import internal.connectors.HasSeriesKeysOnlySupported; import internal.web.RestDriverSupport; +import it.bancaditalia.oss.sdmx.api.Dataflow; +import it.bancaditalia.oss.sdmx.exceptions.SdmxException; +import it.bancaditalia.oss.sdmx.parser.v20.DataflowParser; +import it.bancaditalia.oss.sdmx.parser.v21.Sdmx21Queries; +import it.bancaditalia.oss.sdmx.util.RestQueryBuilder; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; /** * @@ -34,7 +47,48 @@ public final class IloDriver implements SdmxWebDriver, HasCache { private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:ilo:") - .client(ConnectorRestClient.of(ILO::new)) - .entry("ILO", "International Labour Office", "https://www.ilo.org/ilostat/sdmx/ws/rest") + .client(ConnectorRestClient.of(ILO2::new)) + .entry("ILO", "International Labour Office", URL) .build(); + + @SdmxFix(id = "ILO#1", cause = "Fallback to http due to servers redirecting to http") + private static final String URL = "http://www.ilo.org/ilostat/sdmx/ws/rest"; + + private static final class ILO2 extends ILO implements HasSeriesKeysOnlySupported { + + public ILO2() throws URISyntaxException { + } + + @Override + public boolean isSeriesKeysOnlySupported() { + return true; + } + + @Override + public Map getDataflows() throws SdmxException { + URL query; + try { + query = new RestQueryBuilder(endpoint).addPath("dataflow").addPath("ILO").build(); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + return runQuery(new DataflowParser(), query, null) + .stream() + .collect(Collectors.toMap(Dataflow::getId, Function.identity())); + } + + @Override + protected URL buildDataQuery(Dataflow dataflow, String resource, String startTime, String endTime, boolean serieskeysonly, String updatedAfter, boolean includeHistory) throws SdmxException { + try { + return new Sdmx21Queries(endpoint) + .addParams(startTime, endTime, serieskeysonly, updatedAfter, includeHistory, format) + .addPath("data") + .addPath(dataflow.getFullIdentifier()) + .addPath("all".equals(resource) ? "ALL" : resource) + .build(); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } + } } From f9ead124fd579f1c76c67030a53b314c2f656804 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 21 Dec 2017 10:40:12 +0100 Subject: [PATCH 66/68] Improved error reporting. --- .../internal/connectors/ConnectorRestClient.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java index 06fc3358..a50d898a 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java @@ -63,7 +63,7 @@ public static RestClient.Supplier of(ConnectorConstructor supplier) { RestSdmxClient client = supplier.get(); client.setEndpoint(getEndpoint(x, prefix)); configure(client, x.getProperties(), z); - return new ConnectorRestClient(client); + return new ConnectorRestClient(x.getName(), client); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } @@ -75,13 +75,16 @@ public static RestClient.Supplier of(BiFunction, RestSdmxClient> try { RestSdmxClient client = supplier.apply(getEndpoint(x, prefix), x.getProperties()); configure(client, x.getProperties(), z); - return new ConnectorRestClient(client); + return new ConnectorRestClient(x.getName(), client); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } }; } + @lombok.NonNull + private final String name; + @lombok.NonNull private final RestSdmxClient connector; @@ -95,7 +98,7 @@ public List getFlows() throws IOException { .map(Util::toFlow) .collect(Collectors.toList()); } catch (SdmxException ex) { - throw expected(ex, "Failed to get datasets"); + throw expected(ex, "Failed to get dataflows from '%s'", name); } } @@ -104,7 +107,7 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { try { return Util.toFlow(connector.getDataflow(ref.getId(), ref.getAgency(), ref.getVersion())); } catch (SdmxException ex) { - throw expected(ex, "Failed to get details from dataset '%s'", ref); + throw expected(ex, "Failed to get dataflow '%s' from '%s'", ref, name); } } @@ -113,7 +116,7 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { try { return Util.toStructure(connector.getDataFlowStructure(Util.fromStructureRef(ref), true)); } catch (SdmxException ex) { - throw expected(ex, "Failed to get data structure from dataset '%s'", ref); + throw expected(ex, "Failed to get datastructure '%s' from '%s'", ref, name); } } @@ -127,7 +130,7 @@ public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure ds if (Util.isNoResultMatchingQuery(ex)) { return NoOpCursor.noOp(); } - throw expected(ex, "Failed to get data from dataset '%s' with key '%s'", flowRef, query.getKey()); + throw expected(ex, "Failed to get data '%s' with %s from '%s'", flowRef, query, name); } } From 7b6f5eb5523043889acf5c05250ba25e9dcf5809 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 21 Dec 2017 11:46:44 +0100 Subject: [PATCH 67/68] Fixed UIS driver. --- .../connectors/drivers/UisDriver.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java index b05d3100..dddc1315 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java +++ b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java @@ -17,11 +17,14 @@ package internal.connectors.drivers; import be.nbb.sdmx.facade.util.HasCache; -import it.bancaditalia.oss.sdmx.client.custom.UIS; +import be.nbb.sdmx.facade.util.SdmxFix; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; import internal.web.RestDriverSupport; +import it.bancaditalia.oss.sdmx.client.custom.DotStat; +import java.net.URI; +import java.util.Map; /** * @@ -34,7 +37,17 @@ public final class UisDriver implements SdmxWebDriver, HasCache { private final RestDriverSupport support = RestDriverSupport .builder() .prefix("sdmx:uis:") - .client(ConnectorRestClient.of(UIS::new)) - .entry("UIS", "Unesco Institute for Statistics", "http://data.uis.unesco.org/RestSDMX/sdmx.ashx") + .client(ConnectorRestClient.of(UIS2::new)) + .entry("UIS", "Unesco Institute for Statistics", URL) .build(); + + @SdmxFix(id = "#UIS1", cause = "API requires auth by key in header and this is not supported yet in facade") + private final static String URL = "http://data.uis.unesco.org/RestSDMX/sdmx.ashx"; + + private static final class UIS2 extends DotStat { + + private UIS2(URI uri, Map properties) { + super("", uri, false, "compact_v2"); + } + } } From 23d49bf78be64e5a2019af3e09308b1044495d5d Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Thu, 1 Mar 2018 13:25:40 +0100 Subject: [PATCH 68/68] Updated to version 2.2.1 --- demetra-dotstat-core/pom.xml | 4 ++-- demetra-dotstat-desktop/pom.xml | 4 ++-- pom.xml | 4 ++-- sdmx-facade/pom.xml | 6 +++--- sdmx-facade/sdmx-facade-api/pom.xml | 4 ++-- sdmx-facade/sdmx-facade-file/pom.xml | 4 ++-- sdmx-facade/sdmx-facade-samples/pom.xml | 4 ++-- sdmx-facade/sdmx-facade-util/pom.xml | 4 ++-- sdmx-facade/sdmx-facade-web/pom.xml | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml index 9f92e7dd..e1b986f4 100644 --- a/demetra-dotstat-core/pom.xml +++ b/demetra-dotstat-core/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-core - 2.2.1-SNAPSHOT + 2.2.1 jar Demetra - DotStat - Core @@ -119,4 +119,4 @@ https://oss.jfrog.org/artifactory/oss-release-local - \ No newline at end of file + diff --git a/demetra-dotstat-desktop/pom.xml b/demetra-dotstat-desktop/pom.xml index 19f962ea..a2b42993 100644 --- a/demetra-dotstat-desktop/pom.xml +++ b/demetra-dotstat-desktop/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-desktop - 2.2.1-SNAPSHOT + 2.2.1 nbm Demetra - DotStat - Desktop @@ -169,4 +169,4 @@ - \ No newline at end of file + diff --git a/pom.xml b/pom.xml index 362e860e..6353ca1e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-aggregator - 2.2.1-SNAPSHOT + 2.2.1 pom Demetra - DotStat @@ -26,4 +26,4 @@ - \ No newline at end of file + diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index 57f7a393..5be95cce 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -4,7 +4,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 pom @@ -27,7 +27,7 @@ 1.16.18 4.12 3.8.0 - 0.0.1-SNAPSHOT + 0.0.1 @@ -121,4 +121,4 @@ https://oss.jfrog.org/artifactory/oss-snapshot-local - \ No newline at end of file + diff --git a/sdmx-facade/sdmx-facade-api/pom.xml b/sdmx-facade/sdmx-facade-api/pom.xml index 60df5f78..8eed9eb5 100644 --- a/sdmx-facade/sdmx-facade-api/pom.xml +++ b/sdmx-facade/sdmx-facade-api/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 sdmx-facade-api @@ -35,4 +35,4 @@ test - \ No newline at end of file + diff --git a/sdmx-facade/sdmx-facade-file/pom.xml b/sdmx-facade/sdmx-facade-file/pom.xml index 072146fa..a32ca5b8 100644 --- a/sdmx-facade/sdmx-facade-file/pom.xml +++ b/sdmx-facade/sdmx-facade-file/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 sdmx-facade-file @@ -49,4 +49,4 @@ test - \ No newline at end of file + diff --git a/sdmx-facade/sdmx-facade-samples/pom.xml b/sdmx-facade/sdmx-facade-samples/pom.xml index 4511f7b5..e1c9d531 100644 --- a/sdmx-facade/sdmx-facade-samples/pom.xml +++ b/sdmx-facade/sdmx-facade-samples/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 sdmx-facade-samples @@ -39,4 +39,4 @@ true - \ No newline at end of file + diff --git a/sdmx-facade/sdmx-facade-util/pom.xml b/sdmx-facade/sdmx-facade-util/pom.xml index 0007ce57..14e7826e 100644 --- a/sdmx-facade/sdmx-facade-util/pom.xml +++ b/sdmx-facade/sdmx-facade-util/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 sdmx-facade-util @@ -49,4 +49,4 @@ test - \ No newline at end of file + diff --git a/sdmx-facade/sdmx-facade-web/pom.xml b/sdmx-facade/sdmx-facade-web/pom.xml index 0485369b..77e98e41 100644 --- a/sdmx-facade/sdmx-facade-web/pom.xml +++ b/sdmx-facade/sdmx-facade-web/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.1-SNAPSHOT + 2.2.1 sdmx-facade-web @@ -109,4 +109,4 @@ sdmx-facade-web - \ No newline at end of file +