diff --git a/.gitignore b/.gitignore index 51f9d064..945799ca 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ /sdmx-facade/sdmx-facade-util/target/ /sdmx-facade/sdmx-facade-file/target/ /sdmx-facade/sdmx-facade-samples/target/ -/sdmx-facade/sdmx-facade-web/target/ \ No newline at end of file +/sdmx-facade/sdmx-facade-web-ri/target/ +/sdmx-facade/sdmx-facade-util-web/target/ +/sdmx-facade/sdmx-facade-util-xml/target/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 21f8717e..3b00836e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,26 @@ language: java jdk: - oraclejdk8 + - openjdk11 + +env: + global: + - DEPLOY_JDK=oraclejdk8 + - DEPLOY_REPO=nbbrd/jdemetra-dotstat + - DEPLOY_BIN=demetra-dotstat-desktop/target/*.nbm + +deploy: + # Github releases from tags + - provider: releases + api_key: "${GITHUB_KEY}" + skip_cleanup: true + draft: true + file_glob: true + file: "${DEPLOY_BIN}" + on: + tags: true + repo: "${DEPLOY_REPO}" + condition: $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_JDK_VERSION == $DEPLOY_JDK cache: directories: diff --git a/README.md b/README.md index 1685f9b4..0e5d2208 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # SDMX extension for JDemetra+ -This plugin provides time series from [SDMX](https://sdmx.org/) to [JDemetra+](https://github.com/jdemetra/jdemetra-app) by using [SDMX Web Services](https://github.com/sdmx-twg/sdmx-rest/wiki). - [![Download](https://img.shields.io/github/release/nbbrd/jdemetra-dotstat.svg)](https://github.com/nbbrd/jdemetra-dotstat/releases/latest) -## Quickstart +This plugin provides time series from [SDMX](https://sdmx.org/) to [JDemetra+](https://github.com/jdemetra/jdemetra-app) by querying [web services](https://github.com/nbbrd/jdemetra-dotstat/wiki/Supported-web-services) or parsing [files](https://github.com/nbbrd/jdemetra-dotstat/wiki/Supported-file-formats). + +![Browsing web service](https://github.com/nbbrd/jdemetra-dotstat/wiki/assets/browse_web_service.gif) See [documentation](https://github.com/nbbrd/jdemetra-dotstat/wiki). diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml index 1ba5a079..4297ceb6 100644 --- a/demetra-dotstat-core/pom.xml +++ b/demetra-dotstat-core/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-core - 2.2.2 + 2.2.3 jar Demetra - DotStat - Core @@ -27,8 +27,11 @@ UTF-8 1.8 1.8 + 3.8.0 + 2.2.0 + 1.18.4 @@ -40,6 +43,11 @@ pom import + + org.projectlombok + lombok + ${lombok.version} + @@ -73,17 +81,23 @@ be.nbb.sdmx - sdmx-facade-web + sdmx-facade-file ${project.version} + + eu.europa.ec.joinup.sat + demetra-jdbc + + be.nbb.sdmx - sdmx-facade-file + sdmx-facade-connectors ${project.version} - eu.europa.ec.joinup.sat - demetra-jdbc + be.nbb.sdmx + sdmx-facade-web-ri + ${project.version} @@ -109,6 +123,41 @@ + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + + + + + + + + 11 + + + + javax.xml.bind + jaxb-api + 2.4.0-b180830.0359 + provided + + + org.glassfish.jaxb + jaxb-runtime + 2.4.0-b180830.0438 + provided + + + + + netbeans-releases @@ -118,5 +167,12 @@ oss-jfrog-artifactory-releases https://oss.jfrog.org/artifactory/oss-release-local + + projectlombok.org + https://projectlombok.org/edge-releases + + true + + diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java index 3afb3771..afdc1aa7 100644 --- a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java +++ b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java @@ -49,6 +49,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import be.nbb.sdmx.facade.SdmxManager; +import ec.tss.TsAsyncMode; +import ec.tss.tsproviders.cursor.TsCursorAsFiller; +import ec.tss.tsproviders.utils.TsFillerAsProvider; +import java.io.EOFException; /** * @@ -78,6 +82,9 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties { @lombok.experimental.Delegate(excludes = {HasTsCursor.class, HasDataDisplayName.class}) private final CubeSupport cubeSupport; + @lombok.experimental.Delegate + private final HasDataDisplayName dataDisplayName; + @lombok.experimental.Delegate private final ITsProvider tsSupport; @@ -92,7 +99,8 @@ public SdmxFileProvider() { this.beanSupport = HasDataSourceBean.of(NAME, sdmxParam, sdmxParam.getVersion()); this.filePathSupport = HasFilePaths.of(cache::invalidateAll); this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, properties, filePathSupport, sdmxParam)); - this.tsSupport = CubeSupport.asTsProvider(NAME, logger, cubeSupport, monikerSupport, cache::invalidateAll); + this.dataDisplayName = new SdmxFileDataDisplayName(beanSupport, cubeSupport); + this.tsSupport = TsFillerAsProvider.of(NAME, TsAsyncMode.Once, TsCursorAsFiller.of(logger, cubeSupport, monikerSupport, dataDisplayName), cache::invalidateAll); } @Override @@ -110,24 +118,8 @@ public boolean accept(File pathname) { return pathname.getName().toLowerCase().endsWith(".xml"); } - @Override - public String getDisplayName(DataSource dataSource) throws IllegalArgumentException { - return getSourceLabel(decodeBean(dataSource)); - } - - @Override - public String getDisplayName(DataSet dataSet) throws IllegalArgumentException { - return cubeSupport.getDisplayName(dataSet); - } - - @Override - public String getDisplayName(IOException exception) throws IllegalArgumentException { - return cubeSupport.getDisplayName(exception); - } - - @Override - public String getDisplayNodeName(DataSet dataSet) throws IllegalArgumentException { - return cubeSupport.getDisplayNodeName(dataSet); + private static String getSourceLabel(SdmxFileBean bean) { + return bean.getFile().getPath(); } @lombok.AllArgsConstructor @@ -182,7 +174,33 @@ private static IO.Supplier toConnection(HasSdmxProperties proper } } - private static String getSourceLabel(SdmxFileBean bean) { - return bean.getFile().getPath(); + @lombok.AllArgsConstructor + private static final class SdmxFileDataDisplayName implements HasDataDisplayName { + + private final HasDataSourceBean beanSupport; + private final CubeSupport cubeSupport; + + @Override + public String getDisplayName(DataSource dataSource) throws IllegalArgumentException { + return getSourceLabel(beanSupport.decodeBean(dataSource)); + } + + @Override + public String getDisplayName(DataSet dataSet) throws IllegalArgumentException { + return cubeSupport.getDisplayName(dataSet); + } + + @Override + public String getDisplayName(IOException exception) throws IllegalArgumentException { + if (exception instanceof EOFException) { + return "Unexpected end-of-file: " + exception.getMessage(); + } + return cubeSupport.getDisplayName(exception); + } + + @Override + public String getDisplayNodeName(DataSet dataSet) throws IllegalArgumentException { + return cubeSupport.getDisplayNodeName(dataSet); + } } } diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java index 544319e8..9ffa0a68 100644 --- a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java +++ b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.SdmxConnection; import com.google.common.collect.ImmutableList; import ec.tss.tsproviders.cursor.TsCursor; @@ -85,7 +85,7 @@ public List getChildren(SdmxConnection conn, DataflowRef flowRef, Key re // private TsCursor request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException { - return new SdmxDataAdapter(key, conn.getCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute); + return new SdmxDataAdapter(key, conn.getDataCursor(flowRef, key, seriesKeysOnly ? DataFilter.SERIES_KEYS_ONLY : DataFilter.ALL), labelAttribute); } private TsCursor computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException { diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java index e1567f89..82400204 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.repo.SdmxRepositoryManager; import com.google.common.base.Joiner; @@ -108,7 +108,7 @@ public void testGetKey() throws Exception { @Test public void testGetKeyFromTs() throws Exception { assertThat(manager.getConnection("NBB") - .getStream(NBB_FLOW_REF, DataQuery.of(Key.ALL, true)) + .getDataStream(NBB_FLOW_REF, Key.ALL, DataFilter.SERIES_KEYS_ONLY) .map(Series::getKey)).contains(Key.parse("LOCSTL04.AUS.M")); } diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java index 09b781df..976e7b8c 100644 --- a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java +++ b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java @@ -146,12 +146,12 @@ public void testContent() throws IOException { assertThat(newColInfo(p, NO_XML, STRUCT20)).satisfies(info -> { assertThat(p.get(info)).isFalse(); - assertThat(info.invalidDataCause).contains("XMLStreamException"); + assertThat(info.invalidDataCause).contains("end-of-file"); }); assertThat(newColInfo(p, GENERIC20, NO_XML)).satisfies(info -> { assertThat(p.get(info)).isFalse(); - assertThat(info.invalidDataCause).contains("XMLStreamException"); + assertThat(info.invalidDataCause).contains("end-of-file"); }); } } diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java index 3fcde583..4a7769f9 100644 --- a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java +++ b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java @@ -26,6 +26,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; @@ -104,7 +105,7 @@ private List flow20(ByteSource xml, LanguagePriorityList l) throws IOE } List data20(ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd, DataFactory.sdmx20()).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } @@ -118,7 +119,7 @@ private List flow21(ByteSource xml, LanguagePriorityList l) throws IOE } List data21(ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd, DataFactory.sdmx21()).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } diff --git a/demetra-dotstat-desktop/pom.xml b/demetra-dotstat-desktop/pom.xml index 2826f654..e62b1d36 100644 --- a/demetra-dotstat-desktop/pom.xml +++ b/demetra-dotstat-desktop/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-desktop - 2.2.2 + 2.2.3 nbm Demetra - DotStat - Desktop @@ -27,10 +27,13 @@ UTF-8 1.8 1.8 - 2.6 - 3.14 + 3.8.0 + 3.1.1 + 4.1 + 2.2.0 + 1.0.0-SNAPSHOT @@ -134,6 +137,11 @@ sdmx-facade-api ${project.version} + + be.nbb.rd + java-net-proxy + ${java-net-proxy.version} + @@ -145,9 +153,22 @@ oss-jfrog-artifactory-releases https://oss.jfrog.org/artifactory/oss-release-local + + oss-jfrog-artifactory-snapshots + https://oss.jfrog.org/artifactory/oss-snapshot-local + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + org.apache.maven.plugins @@ -155,7 +176,9 @@ ${maven-jar-plugin.version} - true + + ${project.build.outputDirectory}/META-INF/MANIFEST.MF + @@ -172,4 +195,21 @@ + + + + + + 11 + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + + diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java index dc130aa4..df47f413 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProviderBuddy.java @@ -32,7 +32,6 @@ import ec.nbdemetra.ui.tsproviders.IDataSourceProviderBuddy; import ec.tss.tsproviders.DataSet; import ec.tss.tsproviders.IFileLoader; -import ec.tss.tsproviders.TsProviders; import ec.tstoolkit.utilities.GuavaCaches; import internal.file.SdmxFileUtil; import internal.sdmx.SdmxAutoCompletion; @@ -52,6 +51,7 @@ import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.SdmxManager; +import org.openide.util.Lookup; /** * @@ -133,7 +133,7 @@ private static SdmxFileManager createManager() { } private static Optional lookupProvider() { - return TsProviders.lookup(SdmxFileProvider.class, SdmxFileProvider.NAME).toJavaUtil(); + return Optional.ofNullable(Lookup.getDefault().lookup(SdmxFileProvider.class)); } private static Configurator createConfigurator() { diff --git a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java index f18c69f3..a33b5aa2 100644 --- a/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java +++ b/demetra-dotstat-desktop/src/main/java/be/nbb/demetra/sdmx/web/SdmxWebProviderBuddy.java @@ -33,7 +33,6 @@ import ec.nbdemetra.ui.properties.PropertySheetDialogBuilder; import ec.nbdemetra.ui.tsproviders.IDataSourceProviderBuddy; import ec.tss.tsproviders.DataSet; -import ec.tss.tsproviders.TsProviders; import ec.tstoolkit.utilities.GuavaCaches; import internal.sdmx.SdmxAutoCompletion; import java.awt.Image; @@ -50,9 +49,15 @@ import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.SdmxManager; +import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import java.net.ProxySelector; +import java.util.ArrayList; +import java.util.ServiceLoader; import javax.annotation.Nullable; import javax.net.ssl.SSLSocketFactory; +import nbbrd.net.SystemProxySelector; +import org.openide.util.Lookup; /** * @@ -142,7 +147,7 @@ public void setSSLSocketFactory(@Nullable SSLSocketFactory sslSocketFactory) { // private static Optional lookupProvider() { - return TsProviders.lookup(SdmxWebProvider.class, SdmxWebProvider.NAME).toJavaUtil(); + return Optional.ofNullable(Lookup.getDefault().lookup(SdmxWebProvider.class)); } private static Configurator createConfigurator() { @@ -150,8 +155,18 @@ private static Configurator createConfigurator() { } private static SdmxWebManager createManager() { - SdmxWebManager result = SdmxWebManager.ofServiceLoader(); - result.setCache(GuavaCaches.softValuesCacheAsMap()); + ConcurrentMap cache = GuavaCaches.softValuesCacheAsMap(); + List drivers = new ArrayList<>(); + ServiceLoader + .load(SdmxWebDriver.class) + .forEach(o -> { + if (o instanceof HasCache) { + ((HasCache) o).setCache(cache); + } + drivers.add(o); + }); + SdmxWebManager result = SdmxWebManager.of(drivers); + result.setProxySelector(SystemProxySelector.ofServiceLoader()); return result; } diff --git a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java index eab9ac96..2da85d7b 100644 --- a/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java +++ b/demetra-dotstat-desktop/src/main/java/internal/sdmx/SdmxAutoCompletion.java @@ -83,7 +83,11 @@ public AutoCompletionSource onSources(SdmxWebManager manager) { } public ListCellRenderer getSourceRenderer() { - return CustomListCellRenderer.of(SdmxWebSource::getDescription, SdmxWebSource::getName); + return CustomListCellRenderer.of(SdmxAutoCompletion::getNameAndDescription, o -> null); + } + + private String getNameAndDescription(SdmxWebSource o) { + return o.getName() + ": " + o.getDescription(); } public AutoCompletionSource onFlows(SdmxManager manager, Supplier source, ConcurrentMap cache) { @@ -129,7 +133,13 @@ public String getDefaultDimensionsAsString(SdmxManager manager, Supplier private List filterAndSortSources(List allValues, String term) { Predicate filter = ExtAutoCompletionSource.basicFilter(term); - return allValues.stream() + // need to filter out duplicates + return allValues + .stream() + .collect(Collectors.groupingBy(SdmxWebSource::getName)) + .values() + .stream() + .flatMap(o -> o.stream().limit(1)) .filter(o -> filter.test(o.getDescription()) || filter.test(o.getName())) .sorted(Comparator.comparing(SdmxWebSource::getDescription)) .collect(Collectors.toList()); diff --git a/pom.xml b/pom.xml index 664da3b3..0dd93d67 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ be.nbb.demetra demetra-dotstat-aggregator - 2.2.2 + 2.2.3 pom Demetra - DotStat diff --git a/sdmx-facade/pom.xml b/sdmx-facade/pom.xml index ae308b16..99d01937 100644 --- a/sdmx-facade/pom.xml +++ b/sdmx-facade/pom.xml @@ -4,17 +4,20 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 pom sdmx-facade sdmx-facade-api + sdmx-facade-samples sdmx-facade-util + sdmx-facade-util-xml + sdmx-facade-util-web sdmx-facade-file - sdmx-facade-samples - sdmx-facade-web + sdmx-facade-web-ri + sdmx-facade-connectors @@ -22,14 +25,18 @@ UTF-8 1.8 1.8 - 0.7.9 + 3.8.0 + 0.8.2 - - 3.0.2 - 1.16.18 + 4.12 - 3.8.0 - 0.0.1 + 3.0.2 + 3.11.1 + RELEASE82 + 1.18.4 + + + 0.0.2 @@ -60,6 +67,11 @@ java-io-util ${java-io-util.version} + + org.netbeans.api + org-openide-util-lookup + ${netbeans.version} + ${project.groupId} @@ -73,7 +85,12 @@ ${project.groupId} - sdmx-facade-web + sdmx-facade-connectors + ${project.version} + + + ${project.groupId} + sdmx-facade-util-web ${project.version} @@ -86,29 +103,47 @@ sdmx-facade-samples ${project.version} + + ${project.groupId} + sdmx-facade-util-xml + ${project.version} + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + + prepare-agent + + + + report + prepare-package + + report + + + + + + org.jacoco jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - prepare-agent - - - - report - prepare-package - - report - - - @@ -122,5 +157,16 @@ oss-jfrog-artifactory-snapshots https://oss.jfrog.org/artifactory/oss-snapshot-local + + netbeans + http://bits.netbeans.org/maven2/ + + + projectlombok.org + https://projectlombok.org/edge-releases + + true + + diff --git a/sdmx-facade/sdmx-facade-api/pom.xml b/sdmx-facade/sdmx-facade-api/pom.xml index 79843443..614a584d 100644 --- a/sdmx-facade/sdmx-facade-api/pom.xml +++ b/sdmx-facade/sdmx-facade-api/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 sdmx-facade-api diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataFilter.java similarity index 67% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataFilter.java index 456f170c..9c07a4f3 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQuery.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataFilter.java @@ -16,28 +16,26 @@ */ package be.nbb.sdmx.facade; -import javax.annotation.Nonnull; - /** * * @author Philippe Charles */ @lombok.Value @lombok.Builder(builderClassName = "Builder") -public class DataQuery { +public class DataFilter { - @lombok.NonNull - Key key; + public static final DataFilter ALL = builder().build(); + public static final DataFilter SERIES_KEYS_ONLY = builder().detail(Detail.SERIES_KEYS_ONLY).build(); @lombok.NonNull @lombok.Builder.Default - DataQueryDetail detail = DataQueryDetail.FULL; + Detail detail = Detail.FULL; + + public boolean isSeriesKeyOnly() { + return detail.equals(Detail.SERIES_KEYS_ONLY); + } - @Nonnull - public static DataQuery of(@Nonnull Key key, boolean seriesKeysOnly) { - return DataQuery.builder() - .key(key) - .detail(seriesKeysOnly ? DataQueryDetail.SERIES_KEYS_ONLY : DataQueryDetail.FULL) - .build(); + public enum Detail { + FULL, SERIES_KEYS_ONLY } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructure.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructure.java index 6e97ea4c..e90f55d2 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructure.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataStructure.java @@ -28,7 +28,7 @@ * @author Philippe Charles */ @lombok.Value -@lombok.Builder(builderClassName = "Builder") +@lombok.Builder(builderClassName = "Builder", toBuilder = true) public class DataStructure implements HasLabel { /** diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java index 913e31da..0f6e8b6d 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/SdmxConnection.java @@ -18,7 +18,8 @@ import java.io.Closeable; import java.io.IOException; -import java.util.Set; +import java.util.Collection; +import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; @@ -31,7 +32,7 @@ public interface SdmxConnection extends Closeable { @Nonnull - Set getFlows() throws IOException; + Collection getFlows() throws IOException; @Nonnull Dataflow getFlow(@Nonnull DataflowRef flowRef) throws IOException; @@ -40,10 +41,13 @@ public interface SdmxConnection extends Closeable { DataStructure getStructure(@Nonnull DataflowRef flowRef) throws IOException; @Nonnull - DataCursor getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + List getData(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) throws IOException; @Nonnull - Stream getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + Stream getDataStream(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) throws IOException; + + @Nonnull + DataCursor getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java similarity index 94% rename from sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java index 1f5b3996..2f810c7c 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebConnection.java @@ -29,4 +29,7 @@ public interface SdmxWebConnection extends SdmxConnection { @Nonnull Duration ping() throws IOException; + + @Nonnull + String getDriver() throws IOException; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java similarity index 65% rename from sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java index 0fd57132..eed76c94 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebManager.java @@ -19,19 +19,14 @@ import be.nbb.sdmx.facade.web.spi.SdmxWebContext; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.util.UnexpectedIOException; import java.io.IOException; import java.net.ProxySelector; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.ServiceLoader; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -43,6 +38,10 @@ import javax.net.ssl.SSLSocketFactory; import lombok.AccessLevel; import be.nbb.sdmx.facade.SdmxManager; +import java.util.Comparator; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.StreamSupport; /** * @@ -50,7 +49,7 @@ */ @lombok.RequiredArgsConstructor(access = AccessLevel.PRIVATE) @lombok.extern.java.Log -public final class SdmxWebManager implements SdmxManager, HasCache { +public final class SdmxWebManager implements SdmxManager { @Nonnull public static SdmxWebManager ofServiceLoader() { @@ -64,34 +63,30 @@ public static SdmxWebManager of(@Nonnull SdmxWebDriver... drivers) { @Nonnull public static SdmxWebManager of(@Nonnull Iterable drivers) { - List driverList = new ArrayList<>(); - drivers.forEach(driverList::add); - - ConcurrentMap sourceByName = new ConcurrentHashMap<>(); - updateSourceMap(sourceByName, driverList.stream().flatMap(o -> tryGetDefaultSources(o))); + List orderedListOfDrivers = StreamSupport + .stream(drivers.spliterator(), false) + .sorted(Comparator.comparing(SdmxWebDriver::getRank).reversed().thenComparing(SdmxWebDriver::getName)) + .collect(Collectors.toList()); - HasCache cacheSupport = HasCache.of(ConcurrentHashMap::new, (o, n) -> applyCache(n, driverList)); + CopyOnWriteArrayList sources = orderedListOfDrivers + .stream() + .flatMap(SdmxWebManager::tryGetDefaultSources) + .collect(Collectors.toCollection(CopyOnWriteArrayList::new)); - return new SdmxWebManager( - new AtomicReference<>(LanguagePriorityList.ANY), - new AtomicReference<>(SdmxWebContext.builder().build()), - driverList, sourceByName, cacheSupport); + return new SdmxWebManager(orderedListOfDrivers, sources); } - private final AtomicReference languages; - private final AtomicReference context; + private final AtomicReference context = new AtomicReference<>(SdmxWebContext.builder().build()); private final List drivers; - private final ConcurrentMap sourceByName; - private final HasCache cacheSupport; + private final CopyOnWriteArrayList sources; @Override public SdmxWebConnection getConnection(String name) throws IOException { Objects.requireNonNull(name); - SdmxWebSource source = sourceByName.get(name); - if (source == null) { - throw new IOException("Cannot find entry point for '" + name + "'"); - } + SdmxWebSource source = lookupSource(name) + .orElseThrow(() -> new IOException("Cannot find entry point for '" + name + "'")); + return getConnection(source); } @@ -99,33 +94,21 @@ public SdmxWebConnection getConnection(String name) throws IOException { public SdmxWebConnection getConnection(@Nonnull SdmxWebSource source) throws IOException { Objects.requireNonNull(source); - SdmxWebDriver driver = drivers - .stream() - .filter(o -> source.getDriver().equals(o.getName())) - .findFirst() + SdmxWebDriver driver = lookupDriver(source.getDriver()) .orElseThrow(() -> new IOException("Failed to find a suitable driver for '" + source + "'")); - return tryConnect(driver, source, languages.get(), context.get()); + return tryConnect(driver, source, context.get()); } @Override public LanguagePriorityList getLanguages() { - return languages.get(); + return context.get().getLanguages(); } @Override public void setLanguages(LanguagePriorityList languages) { - this.languages.set(languages != null ? languages : LanguagePriorityList.ANY); - } - - @Override - public ConcurrentMap getCache() { - return cacheSupport.getCache(); - } - - @Override - public void setCache(ConcurrentMap cache) { - this.cacheSupport.setCache(cache); + LanguagePriorityList newObj = languages != null ? languages : LanguagePriorityList.ANY; + this.context.set(context.get().toBuilder().languages(newObj).build()); } @Nonnull @@ -160,11 +143,12 @@ public void setLogger(@Nullable Logger logger) { @Nonnull public List getSources() { - return new ArrayList<>(sourceByName.values()); + return Collections.unmodifiableList(sources); } public void setSources(@Nonnull List list) { - updateSourceMap(sourceByName, list.stream()); + sources.clear(); + sources.addAll(list); } @Nonnull @@ -178,38 +162,39 @@ public List getDrivers() { @Nonnull public Collection getSupportedProperties(@Nonnull String driver) { - return drivers - .stream() - .filter(o -> driver.equals(o.getName())) + Objects.requireNonNull(driver); + return lookupDriver(driver) .map(SdmxWebDriver::getSupportedProperties) - .findFirst() .orElse(Collections.emptyList()); } - private static void applyCache(ConcurrentMap cache, List drivers) { - drivers.stream() - .filter(HasCache.class::isInstance) - .forEach(o -> ((HasCache) o).setCache(cache)); + private Optional lookupSource(String name) { + return sources + .stream() + .filter(o -> name.equals(o.getName())) + .findFirst(); } - private static void updateSourceMap(ConcurrentMap sourceByName, Stream list) { - sourceByName.clear(); - list.forEach(o -> sourceByName.put(o.getName(), o)); + private Optional lookupDriver(String name) { + return drivers + .stream() + .filter(o -> name.equals(o.getName())) + .findFirst(); } @SuppressWarnings("null") - private static SdmxWebConnection tryConnect(SdmxWebDriver driver, SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) throws IOException { + private static SdmxWebConnection tryConnect(SdmxWebDriver driver, SdmxWebSource s, SdmxWebContext c) throws IOException { SdmxWebConnection result; try { - result = driver.connect(s, l, c); + result = driver.connect(s, c); } catch (RuntimeException ex) { - log.log(Level.WARNING, "Unexpected exception while connecting", ex); - throw new UnexpectedIOException(ex); + c.getLogger().log(Level.WARNING, "Unexpected exception while connecting", ex); + throw new IOException(ex); } if (result == null) { - log.log(Level.WARNING, "Unexpected null connection"); + c.getLogger().log(Level.WARNING, "Unexpected null connection"); throw new IOException("Unexpected null connection"); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebSource.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebSource.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/SdmxWebSource.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/SdmxWebSource.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java similarity index 89% rename from sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java index 2008f20e..7619c61b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebContext.java @@ -16,6 +16,7 @@ */ package be.nbb.sdmx.facade.web.spi; +import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebManager; import java.net.ProxySelector; import java.util.logging.Logger; @@ -30,6 +31,10 @@ @lombok.Builder(builderClassName = "Builder", toBuilder = true) public class SdmxWebContext { + @lombok.NonNull + @lombok.Builder.Default + LanguagePriorityList languages = LanguagePriorityList.ANY; + @lombok.NonNull @lombok.Builder.Default ProxySelector proxySelector = ProxySelector.getDefault(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java similarity index 92% rename from sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java rename to sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java index 6da48560..9fd1b245 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java +++ b/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/web/spi/SdmxWebDriver.java @@ -16,7 +16,6 @@ */ package be.nbb.sdmx.facade.web.spi; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebConnection; import be.nbb.sdmx.facade.web.SdmxWebSource; import java.io.IOException; @@ -34,10 +33,11 @@ public interface SdmxWebDriver { @Nonnull String getName(); + int getRank(); + @Nonnull SdmxWebConnection connect( @Nonnull SdmxWebSource source, - @Nonnull LanguagePriorityList languages, @Nonnull SdmxWebContext context ) throws IOException, IllegalArgumentException; @@ -46,4 +46,7 @@ SdmxWebConnection connect( @Nonnull Collection getSupportedProperties(); + + static final int NATIVE_RANK = Byte.MAX_VALUE; + static final int WRAPPED_RANK = 0; } diff --git a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java index 8b004017..347f0e07 100644 --- a/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java +++ b/sdmx-facade/sdmx-facade-api/src/test/java/be/nbb/sdmx/facade/DimentionTest.java @@ -30,10 +30,11 @@ public class DimentionTest { final String someLabel = "Dim 1"; @Test + @SuppressWarnings("null") public void testBuilder() { - assertThatNullPointerException().isThrownBy(builder()::build).withMessage("id"); - assertThatNullPointerException().isThrownBy(builder().id(null)::build).withMessage("id"); - assertThatNullPointerException().isThrownBy(builder().id(someId).label(null)::build).withMessage("label"); + assertThatNullPointerException().isThrownBy(() -> builder().build()).withMessageContaining("id"); + assertThatNullPointerException().isThrownBy(() -> builder().id(null).build()).withMessageContaining("id"); + assertThatNullPointerException().isThrownBy(() -> builder().id(someId).label(null).build()).withMessageContaining("label"); assertThatNullPointerException().isThrownBy(() -> builder().codes(null)); assertThatExceptionOfType(UnsupportedOperationException.class) diff --git a/sdmx-facade/sdmx-facade-web/pom.xml b/sdmx-facade/sdmx-facade-connectors/pom.xml similarity index 79% rename from sdmx-facade/sdmx-facade-web/pom.xml rename to sdmx-facade/sdmx-facade-connectors/pom.xml index f7630c0a..de80c226 100644 --- a/sdmx-facade/sdmx-facade-web/pom.xml +++ b/sdmx-facade/sdmx-facade-connectors/pom.xml @@ -5,23 +5,19 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 - sdmx-facade-web + sdmx-facade-connectors jar - 9244dfa854 - RELEASE802 + + d08affb9df - - netbeans - http://bits.netbeans.org/maven2/ - jitpack.io https://jitpack.io @@ -35,11 +31,6 @@ SDMX ${sdmx-connectors.version} - - org.netbeans.api - org-openide-util-lookup - ${netbeans.version} - @@ -69,6 +60,10 @@ ${project.groupId} sdmx-facade-util + + ${project.groupId} + sdmx-facade-util-web + com.github.amattioc SDMX @@ -89,5 +84,10 @@ sdmx-facade-samples test + + ${project.groupId} + sdmx-facade-util-xml + test + diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorRestClient.java similarity index 76% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorRestClient.java index aa7385f1..8b1201a6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/ConnectorRestClient.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorRestClient.java @@ -17,14 +17,11 @@ package internal.connectors; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataQueryDetail; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.parser.ObsParser; +import be.nbb.sdmx.facade.parser.DataFactory; import static internal.web.SdmxWebProperty.*; import be.nbb.sdmx.facade.util.NoOpCursor; import java.io.IOException; @@ -45,6 +42,8 @@ import java.util.Arrays; import java.util.Collections; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import internal.web.DataRequest; +import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; /** * @@ -68,13 +67,13 @@ public interface GenericSupplier { } @Nonnull - public static SdmxWebClient.Supplier of(@Nonnull SpecificSupplier supplier) { - return (source, langs, context) -> { + public static SdmxWebClient.Supplier of(@Nonnull SpecificSupplier supplier, @Nonnull DataFactory dataFactory) { + return (source, context) -> { try { RestSdmxClient client = supplier.get(); client.setEndpoint(source.getEndpoint().toURI()); - configure(client, source.getProperties(), langs, context); - return new ConnectorRestClient(source.getName(), client); + configure(client, source.getProperties(), context); + return new ConnectorRestClient(source.getName(), client, dataFactory); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } @@ -82,12 +81,12 @@ public static SdmxWebClient.Supplier of(@Nonnull SpecificSupplier supplier) { } @Nonnull - public static SdmxWebClient.Supplier of(@Nonnull GenericSupplier supplier) { - return (source, langs, context) -> { + public static SdmxWebClient.Supplier of(@Nonnull GenericSupplier supplier, @Nonnull DataFactory dataFactory) { + return (source, context) -> { try { RestSdmxClient client = supplier.get(source.getEndpoint().toURI(), source.getProperties()); - configure(client, source.getProperties(), langs, context); - return new ConnectorRestClient(source.getName(), client); + configure(client, source.getProperties(), context); + return new ConnectorRestClient(source.getName(), client, dataFactory); } catch (URISyntaxException ex) { throw new RuntimeException(ex); } @@ -100,6 +99,14 @@ public static SdmxWebClient.Supplier of(@Nonnull GenericSupplier supplier) { @lombok.NonNull private final RestSdmxClient connector; + @lombok.NonNull + private final DataFactory dataFactory; + + @Override + public String getName() throws IOException { + return name; + } + @Override public List getFlows() throws IOException { try { @@ -133,16 +140,15 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { try { - return connector instanceof HasDataCursor - ? getCursor((HasDataCursor) connector, flowRef, dsd, query) - : getAdaptedCursor(connector, flowRef, dsd, query); + List> data = getData(connector, request, dsd); + return PortableTimeSeriesCursor.of(data, dataFactory, dsd); } catch (SdmxException ex) { if (Connectors.isNoResultMatchingQuery(ex)) { return NoOpCursor.noOp(); } - throw wrap(ex, "Failed to get data '%s' with %s from '%s'", flowRef, query, name); + throw wrap(ex, "Failed to get data '%s' from '%s'", request, name); } } @@ -175,28 +181,24 @@ public Duration ping() throws IOException { READ_TIMEOUT_PROPERTY )); - private static DataCursor getCursor(HasDataCursor connector, DataflowRef flowRef, DataStructure dsd, DataQuery query) throws SdmxException, IOException { - return connector.getDataCursor(flowRef, dsd, query.getKey(), isSeriesKeyOnly(query)); - } - - private static DataCursor getAdaptedCursor(RestSdmxClient connector, DataflowRef flowRef, DataStructure dsd, DataQuery query) throws SdmxException { - return new PortableTimeSeriesCursor(connector.getTimeSeries(Connectors.fromFlowQuery(flowRef, dsd.getRef()), Connectors.fromStructure(dsd), query.getKey().toString(), null, null, isSeriesKeyOnly(query), null, false), ObsParser.standard()); - } - - private static boolean isSeriesKeyOnly(DataQuery query) { - return query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); + private static List> getData(RestSdmxClient connector, DataRequest request, DataStructure dsd) throws SdmxException { + return connector.getTimeSeries(Connectors.fromFlowQuery(request.getFlowRef(), dsd.getRef()), Connectors.fromStructure(dsd), request.getKey().toString(), null, null, request.getFilter().isSeriesKeyOnly(), null, false); } private static IOException wrap(SdmxException ex, String format, Object... args) { return new IOException(String.format(format, args), ex); } - private static void configure(RestSdmxClient client, Map info, LanguagePriorityList langs, SdmxWebContext context) { - client.setLanguages(Connectors.fromLanguages(langs)); + private static void configure(RestSdmxClient client, Map info, SdmxWebContext context) { + client.setLanguages(Connectors.fromLanguages(context.getLanguages())); client.setConnectTimeout(getConnectTimeout(info)); client.setReadTimeout(getReadTimeout(info)); client.setProxySelector(context.getProxySelector()); client.setSslSocketFactory(context.getSslSocketFactory()); // TODO: maxRedirections } + + static { + ConnectorsConfigFix.fixConfiguration(); + } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Connectors.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/Connectors.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/Connectors.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/Connectors.java diff --git a/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorsConfigFix.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorsConfigFix.java new file mode 100644 index 00000000..8d54c0bc --- /dev/null +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/ConnectorsConfigFix.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.connectors; + +import it.bancaditalia.oss.sdmx.util.Configuration; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +class ConnectorsConfigFix { + + static void fixConfiguration() { +// Logger.getLogger(ConnectorRestClient.class.getName()) +// .log(Level.FINE, "Connectors config fix: before"); + + Handler handler = addVoidHandlerBeforeInit(); + Map sysProps = backupSystemPropertiesBeforeInit(); + Configuration.getConfiguration(); + restoreSystemPropertiesAfterInit(sysProps); + removeVoidHandlerAfterInit(handler); + +// Logger.getLogger(ConnectorRestClient.class.getName()) +// .log(Level.FINE, "Connectors config fix: after"); + } + + private static Handler addVoidHandlerBeforeInit() { + Handler result = new VoidHandler(); + Logger logger = Logger.getLogger("SDMX"); + logger.setUseParentHandlers(false); + logger.addHandler(result); + return result; + } + + private static void removeVoidHandlerAfterInit(Handler handler) { + Logger logger = Logger.getLogger("SDMX"); + logger.removeHandler(handler); + logger.setUseParentHandlers(true); + } + + private static Map backupSystemPropertiesBeforeInit() { + return new HashMap<>(System.getProperties()); + } + + private static void restoreSystemPropertiesAfterInit(Map backup) { + System.getProperties() + .keySet() + .stream() + .filter(o -> !backup.containsKey(o)) + .collect(Collectors.toList()) + .forEach(System.getProperties()::remove); + System.getProperties().putAll(backup); + } + + private static final class VoidHandler extends Handler { + + @Override + public void publish(LogRecord lr) { + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/HasSeriesKeysOnlySupported.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/PortableTimeSeriesCursor.java similarity index 68% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/PortableTimeSeriesCursor.java index 90665a47..2d27b6b7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/PortableTimeSeriesCursor.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/PortableTimeSeriesCursor.java @@ -17,8 +17,10 @@ package internal.connectors; import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.parser.ObsParser; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; import java.io.IOException; @@ -28,35 +30,38 @@ import java.util.Map; import java.util.Objects; import be.nbb.sdmx.facade.parser.Freqs; -import static be.nbb.sdmx.facade.parser.Freqs.TIME_FORMAT_CONCEPT; /** * * @author Philippe Charles */ +@lombok.RequiredArgsConstructor public final class PortableTimeSeriesCursor implements DataCursor { - private final Iterator> data; - private final ObsParser obs; - private PortableTimeSeries current; - private int index; - private boolean closed; - private boolean hasObs; - - public PortableTimeSeriesCursor(List> data, ObsParser obs) { - this.data = data.iterator(); - this.obs = obs; - this.closed = false; - this.hasObs = false; + public static PortableTimeSeriesCursor of(List> data, DataFactory df, DataStructure dsd) { + ObsParser obsParser = new ObsParser(df::getPeriodParser, df.getValueParser()); + Freqs.Parser freqParser = df.getFreqParser(dsd); + return new PortableTimeSeriesCursor(data.iterator(), Key.builder(dsd), obsParser, freqParser); } + private final Iterator> data; + private final Key.Builder keyBuilder; + private final ObsParser obsParser; + private final Freqs.Parser freqParser; + + private PortableTimeSeries current = null; + private int index = -1; + private boolean closed = false; + private boolean hasObs = false; + @Override public boolean nextSeries() throws IOException { checkState(); boolean result = data.hasNext(); if (result) { current = data.next(); - obs.frequency(getFrequency(current)); + current.getDimensionsMap().forEach(keyBuilder::put); + obsParser.frequency(freqParser.parse(keyBuilder, current::getAttribute)); index = -1; } else { current = null; @@ -74,13 +79,13 @@ public boolean nextObs() throws IOException { @Override public Key getSeriesKey() throws IOException { checkSeriesState(); - return parseKey(current); + return keyBuilder.build(); } @Override public Frequency getSeriesFrequency() throws IOException { checkSeriesState(); - return obs.getFrequency(); + return obsParser.getFrequency(); } @Override @@ -95,13 +100,16 @@ public Map getSeriesAttributes() throws IOException { checkSeriesState(); Map result = current.getAttributesMap(); result.remove(PortableTimeSeries.GENERATEDNAME_ATTR_NAME); + result.remove(ACTION_ATTR_NAME); + result.remove(VALID_FROM_ATTR_NAME); + result.remove(VALID_TO_ATTR_NAME); return result; } @Override public LocalDateTime getObsPeriod() throws IOException { checkObsState(); - return obs.period(current.get(index).getTimeslot()).parsePeriod(); + return obsParser.period(current.get(index).getTimeslot()).parsePeriod(); } @Override @@ -136,24 +144,7 @@ private void checkObsState() throws IOException, IllegalStateException { } } - // - private static Frequency getFrequency(PortableTimeSeries input) { - if (input.getFrequency() != null) { - return Freqs.parseByFreq(input.getFrequency()); - } - String value = input.getAttribute(TIME_FORMAT_CONCEPT); - if (value != null) { - return Freqs.parseByTimeFormat(value); - } - return Frequency.UNDEFINED; - } - - private static Key parseKey(PortableTimeSeries ts) throws IOException { - Key result = Key.of(ts.getDimensionsMap().values()); - if (!result.isSeries()) { - throw new IOException("Invalid series key '" + result + "'"); - } - return result; - } - // + private static final String ACTION_ATTR_NAME = "action"; + private static final String VALID_FROM_ATTR_NAME = "validFromDate"; + private static final String VALID_TO_ATTR_NAME = "validToDate"; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/AbsDriver.java similarity index 88% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/AbsDriver.java index e0a9a18f..503fa904 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/AbsDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/AbsDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.ABS; import org.openide.util.lookup.ServiceProvider; @@ -33,8 +34,9 @@ public final class AbsDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("abs@connectors") - .client(ConnectorRestClient.of(ABS::new)) + .name("connectors:abs") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(ABS::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .sourceOf("ABS", "Australian Bureau of Statistics", "http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/EurostatDriver.java similarity index 88% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/EurostatDriver.java index b8c6ae42..6b87a587 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/EurostatDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/EurostatDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.EUROSTAT; import org.openide.util.lookup.ServiceProvider; @@ -33,8 +34,9 @@ public final class EurostatDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("eurostat@connectors") - .client(ConnectorRestClient.of(EUROSTAT::new)) + .name("connectors:eurostat") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(EUROSTAT::new, DataFactory.sdmx21())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .sourceOf("EUROSTAT", "Eurostat", "http://ec.europa.eu/eurostat/SDMX/diss-web/rest") .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/IloDriver.java similarity index 82% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/IloDriver.java index 98fb7961..6fc071fb 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/IloDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/IloDriver.java @@ -16,15 +16,17 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxFix; -import it.bancaditalia.oss.sdmx.client.custom.ILO; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.ENDPOINT; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; import internal.connectors.HasSeriesKeysOnlySupported; import internal.web.SdmxWebDriverSupport; import it.bancaditalia.oss.sdmx.api.Dataflow; +import it.bancaditalia.oss.sdmx.client.custom.ILO_Legacy; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; import it.bancaditalia.oss.sdmx.parser.v20.DataflowParser; import it.bancaditalia.oss.sdmx.parser.v21.Sdmx21Queries; @@ -46,16 +48,17 @@ public final class IloDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("ilo@connectors") - .client(ConnectorRestClient.of(ILO2::new)) + .name("connectors:ilo") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(ILO2::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) - .sourceOf("ILO", "International Labour Office", FALLBACK_URL) + .sourceOf("ILO", "International Labour Office", FALLBACK_ENDPOINT) .build(); - @SdmxFix(id = "ILO#1", cause = "Fallback to http due to servers redirecting to http") - private static final String FALLBACK_URL = "http://www.ilo.org/ilostat/sdmx/ws/rest"; + @SdmxFix(id = 1, category = ENDPOINT, cause = "Fallback to http due to servers redirecting to http") + private static final String FALLBACK_ENDPOINT = "http://www.ilo.org/ilostat/sdmx/ws/rest"; - private static final class ILO2 extends ILO implements HasSeriesKeysOnlySupported { + private static final class ILO2 extends ILO_Legacy implements HasSeriesKeysOnlySupported { public ILO2() throws URISyntaxException { } @@ -73,7 +76,7 @@ public Map getDataflows() throws SdmxException { } catch (MalformedURLException ex) { throw new RuntimeException(ex); } - return runQuery(new DataflowParser(), query, null) + return runQuery(new DataflowParser(), query, null, null) .stream() .collect(Collectors.toMap(Dataflow::getId, Function.identity())); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/ImfDriver.java similarity index 88% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/ImfDriver.java index f4f39a6d..af5d1217 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/ImfDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/ImfDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; @@ -33,8 +34,9 @@ public final class ImfDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("imf@connectors") - .client(ConnectorRestClient.of(IMF2::new)) + .name("connectors:imf") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(IMF2::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .sourceOf("IMF", "International Monetary Fund", "http://dataservices.imf.org/REST/SDMX_XML.svc") .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/InseeDriver.java similarity index 57% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/InseeDriver.java index 811004ca..66a57776 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/InseeDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/InseeDriver.java @@ -16,36 +16,24 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.parser.spi.SdmxDialect; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxFix; -import be.nbb.sdmx.facade.util.SdmxMediaType; -import be.nbb.sdmx.facade.util.SeriesSupport; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.CONTENT; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.ENDPOINT; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; -import java.io.IOException; -import java.util.List; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; -import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.Connectors; -import internal.org.springframework.util.xml.XMLEventStreamReader; import internal.util.drivers.InseeDialect; import internal.web.SdmxWebDriverSupport; import it.bancaditalia.oss.sdmx.api.Codelist; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; import it.bancaditalia.oss.sdmx.api.Dimension; -import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; import java.util.Map; import java.util.logging.Level; @@ -54,30 +42,29 @@ * * @author Philippe Charles */ -@lombok.extern.java.Log @ServiceProvider(service = SdmxWebDriver.class) public final class InseeDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("insee@connectors") - .client(ConnectorRestClient.of(InseeClient::new)) + .name("connectors:insee") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(InseeClient::new, DIALECT)) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) - .sourceOf("INSEE", "Institut national de la statistique et des études économiques", FALLBACK_URL) + .sourceOf("INSEE", "Institut national de la statistique et des études économiques", FALLBACK_ENDPOINT) .build(); - @SdmxFix(id = "INSEE#1", cause = "Fallback to http due to some servers that use root certificate unknown to jdk'") - private static final String FALLBACK_URL = "http://bdm.insee.fr/series/sdmx"; + @SdmxFix(id = 1, category = ENDPOINT, cause = "Fallback to http due to some servers that use root certificate unknown to jdk'") + private static final String FALLBACK_ENDPOINT = "http://bdm.insee.fr/series/sdmx"; - private final static class InseeClient extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { + @SdmxFix(id = 2, category = CONTENT, cause = "Does not follow sdmx standard codes") + private static final SdmxDialect DIALECT = new InseeDialect(); - @SdmxFix(id = "INSEE#2", cause = "Does not follow sdmx standard codes") - private final SdmxDialect dialect; + private final static class InseeClient extends RestSdmxClient implements HasSeriesKeysOnlySupported { private InseeClient(URI endpoint, Map properties) { super("", endpoint, false, false, true); - this.dialect = new InseeDialect(); } @Override @@ -88,18 +75,12 @@ public DataFlowStructure getDataFlowStructure(DSDIdentifier dsd, boolean full) t return result; } - @Override - public DataCursor getDataCursor(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { - // FIXME: avoid in-memory copy - return SeriesSupport.asCursor(getData(flowRef, dsd, resource, serieskeysonly), resource); - } - @Override public boolean isSeriesKeysOnlySupported() { return true; } - @SdmxFix(id = "INSEE#4", cause = "Some dimension/code ids are invalid") + @SdmxFix(id = 3, category = CONTENT, cause = "Some dimension/code ids are invalid") private void fixIds(DataFlowStructure dsd) { for (Dimension d : dsd.getDimensions()) { if (d.getId().endsWith("6")) { @@ -113,11 +94,11 @@ private String getValidId(String id) { return id.substring(0, id.length() - 1); } - @SdmxFix(id = "INSEE#3", cause = "Some codes are missing in dsd even when requested with 'references=children'") + @SdmxFix(id = 4, category = CONTENT, cause = "Some codes are missing in dsd even when requested with 'references=children'") private void fixMissingCodes(DataFlowStructure dsd) throws SdmxException { for (Dimension d : dsd.getDimensions()) { Codelist codelist = d.getCodeList(); - if (codelist.getCodes().isEmpty()) { + if (codelist.isEmpty()) { loadMissingCodes(codelist); } } @@ -130,25 +111,8 @@ private void loadMissingCodes(Codelist codelist) throws SdmxException { if (!Connectors.isNoResultMatchingQuery(ex)) { throw ex; } - log.log(Level.WARNING, "Cannot retrieve codes for ''{0}''", codelist.getFullIdentifier()); + logger.log(Level.WARNING, "Cannot retrieve codes for ''{0}''", codelist.getFullIdentifier()); } } - - private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { - return runQuery(getCompactData21Parser(dsd), - buildDataQuery(Connectors.fromFlowQuery(flowRef, dsd.getRef()), resource.toString(), null, null, serieskeysonly, null, false), - SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); - } - - private Parser> getCompactData21Parser(DataStructure dsd) { - return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(dsd, dialect).parse(new XMLEventStreamReader(r), () -> { - })) { - return SeriesSupport.copyOf(cursor); - } catch (IOException ex) { - throw new SdmxIOException("Cannot parse compact data 21", ex); - } - }; - } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/NbbDriver.java similarity index 82% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/NbbDriver.java index b2874c52..a78569fe 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/NbbDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/NbbDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.NBB; import org.openide.util.lookup.ServiceProvider; @@ -33,9 +34,10 @@ public final class NbbDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("nbb@connectors") - .client(ConnectorRestClient.of(NBB::new)) + .name("connectors:nbb") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(NBB::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) - .sourceOf("NBB", "National Bank Belgium", "https://stat.nbb.be/restsdmx/sdmx.ashx") + .sourceOf("NBB", "National Bank of Belgium", "https://stat.nbb.be/restsdmx/sdmx.ashx") .build(); } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/OecdDriver.java similarity index 88% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/OecdDriver.java index 4609c1ef..c3923b27 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/OecdDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/OecdDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.OECD; import org.openide.util.lookup.ServiceProvider; @@ -33,8 +34,9 @@ public final class OecdDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("oecd@connectors") - .client(ConnectorRestClient.of(OECD::new)) + .name("connectors:oecd") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(OECD::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .sourceOf("OECD", "The Organisation for Economic Co-operation and Development", "https://stats.oecd.org/restsdmx/sdmx.ashx") .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx20Driver.java similarity index 90% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx20Driver.java index 2a6ce234..7b62d642 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx20Driver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx20Driver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import static internal.connectors.Connectors.*; import be.nbb.sdmx.facade.util.HasCache; import it.bancaditalia.oss.sdmx.client.custom.RestSdmx20Client; @@ -37,8 +38,9 @@ public final class Sdmx20Driver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("sdmx20@connectors") - .client(ConnectorRestClient.of(Sdmx20Client::new)) + .name("connectors:sdmx20") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(Sdmx20Client::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .supportedProperty(NEEDS_CREDENTIALS_PROPERTY) .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx21Driver.java similarity index 56% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx21Driver.java index 0b1b9b0e..680eaa7b 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/Sdmx21Driver.java @@ -16,34 +16,20 @@ */ package internal.connectors.drivers; -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.util.HasCache; -import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.util.SdmxMediaType; -import be.nbb.sdmx.facade.util.SeriesSupport; +import be.nbb.sdmx.facade.parser.DataFactory; import static internal.web.SdmxWebProperty.*; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import it.bancaditalia.oss.sdmx.client.RestSdmxClient; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException; -import java.io.IOException; -import java.util.List; import java.util.Map; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; -import internal.connectors.HasDataCursor; import internal.connectors.HasSeriesKeysOnlySupported; import internal.connectors.Connectors; import static internal.connectors.Connectors.*; -import internal.org.springframework.util.xml.XMLEventStreamReader; import internal.util.drivers.SdmxWebResource; import internal.web.SdmxWebDriverSupport; import internal.web.SdmxWebProperty; -import it.bancaditalia.oss.sdmx.client.Parser; import java.net.URI; /** @@ -56,8 +42,9 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("sdmx21@connectors") - .client(ConnectorRestClient.of(Sdmx21Client::new)) + .name("connectors:sdmx21") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(Sdmx21Client::new, DataFactory.sdmx21())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .supportedProperty(NEEDS_CREDENTIALS_PROPERTY) .supportedProperty(NEEDS_URL_ENCODING_PROPERTY) @@ -66,7 +53,7 @@ public final class Sdmx21Driver implements SdmxWebDriver, HasCache { .sources(SdmxWebResource.load("/internal/connectors/drivers/sdmx21.xml")) .build(); - private final static class Sdmx21Client extends RestSdmxClient implements HasDataCursor, HasSeriesKeysOnlySupported { + private final static class Sdmx21Client extends RestSdmxClient implements HasSeriesKeysOnlySupported { private final boolean seriesKeysOnlySupported; @@ -78,32 +65,9 @@ private Sdmx21Client(URI endpoint, Map p) { this.seriesKeysOnlySupported = SdmxWebProperty.isSeriesKeysOnlySupported(p); } - @Override - public DataCursor getDataCursor(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException, IOException { - // FIXME: avoid in-memory copy - return SeriesSupport.asCursor(getData(flowRef, dsd, resource, serieskeysonly), resource); - } - @Override public boolean isSeriesKeysOnlySupported() { return seriesKeysOnlySupported; } - - private List getData(DataflowRef flowRef, DataStructure dsd, Key resource, boolean serieskeysonly) throws SdmxException { - return runQuery(getCompactData21Parser(dsd), - buildDataQuery(Connectors.fromFlowQuery(flowRef, dsd.getRef()), resource.toString(), null, null, serieskeysonly, null, false), - SdmxMediaType.STRUCTURE_SPECIFIC_DATA_21); - } - - private Parser> getCompactData21Parser(DataStructure dsd) { - return (r, l) -> { - try (DataCursor cursor = SdmxXmlStreams.compactData21(dsd).parse(new XMLEventStreamReader(r), () -> { - })) { - return SeriesSupport.copyOf(cursor); - } catch (IOException ex) { - throw new SdmxIOException("Cannot parse compact data 21", ex); - } - }; - } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/SeDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/SeDriver.java similarity index 90% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/SeDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/SeDriver.java index c0a86399..c20a52b6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/SeDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/SeDriver.java @@ -16,6 +16,7 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; @@ -35,8 +36,9 @@ public final class SeDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("es@connectors") - .client(ConnectorRestClient.of(EsClient::new)) + .name("connectors:es") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(EsClient::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) .sourceOf("SE", "Statistics Estonia", "http://andmebaas.stat.ee/restsdmx/sdmx.ashx") .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/UisDriver.java similarity index 77% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java rename to sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/UisDriver.java index fcef8708..96ae53df 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/drivers/UisDriver.java +++ b/sdmx-facade/sdmx-facade-connectors/src/main/java/internal/connectors/drivers/UisDriver.java @@ -16,8 +16,10 @@ */ package internal.connectors.drivers; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.util.SdmxFix; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.ENDPOINT; import org.openide.util.lookup.ServiceProvider; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.connectors.ConnectorRestClient; @@ -36,14 +38,15 @@ public final class UisDriver implements SdmxWebDriver, HasCache { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("uis@connectors") - .client(ConnectorRestClient.of(UIS2::new)) + .name("connectors:uis") + .rank(WRAPPED_RANK) + .client(ConnectorRestClient.of(UIS2::new, DataFactory.sdmx20())) .supportedProperties(ConnectorRestClient.CONNECTION_PROPERTIES) - .sourceOf("UIS", "Unesco Institute for Statistics", FALLBACK_URL) + .sourceOf("UIS", "Unesco Institute for Statistics", FALLBACK_ENDPOINT) .build(); - @SdmxFix(id = "#UIS1", cause = "API requires auth by key in header and this is not supported yet in facade") - private final static String FALLBACK_URL = "http://data.uis.unesco.org/RestSDMX/sdmx.ashx"; + @SdmxFix(id = 1, category = ENDPOINT, cause = "API requires auth by key in header and this is not supported yet in facade") + private final static String FALLBACK_ENDPOINT = "http://data.uis.unesco.org/RestSDMX/sdmx.ashx"; private static final class UIS2 extends DotStat { diff --git a/sdmx-facade/sdmx-facade-web/src/main/resources/internal/connectors/drivers/sdmx21.xml b/sdmx-facade/sdmx-facade-connectors/src/main/resources/internal/connectors/drivers/sdmx21.xml similarity index 86% rename from sdmx-facade/sdmx-facade-web/src/main/resources/internal/connectors/drivers/sdmx21.xml rename to sdmx-facade/sdmx-facade-connectors/src/main/resources/internal/connectors/drivers/sdmx21.xml index 02865615..054166c1 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/resources/internal/connectors/drivers/sdmx21.xml +++ b/sdmx-facade/sdmx-facade-connectors/src/main/resources/internal/connectors/drivers/sdmx21.xml @@ -3,7 +3,7 @@ ECB European Central Bank - sdmx21@connectors + connectors:sdmx21 https://sdw-wsrest.ecb.europa.eu/service @@ -11,27 +11,27 @@ INEGI Instituto Nacional de Estadistica y Geografia - sdmx21@connectors + connectors:sdmx21 http://sdmx.snieg.mx/service/Rest WB World Bank - sdmx21@connectors + connectors:sdmx21 https://api.worldbank.org/v2/sdmx/rest UNDATA Data access system to UN databases - sdmx21@connectors + connectors:sdmx21 http://data.un.org/WS/rest ISTAT Istituto Nazionale di Statistica - sdmx21@connectors + connectors:sdmx21 http://sdmx.istat.it/SDMXWS/rest @@ -39,13 +39,13 @@ WITS World Integrated Trade Solutions - sdmx21@connectors + connectors:sdmx21 http://wits.worldbank.org/API/V1/SDMX/V21/rest IMF_SDMX_CENTRAL International Monetary Fund SDMX Central - sdmx21@connectors + connectors:sdmx21 https://sdmxcentral.imf.org/ws/public/sdmxapi/rest diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/CacheAssertions.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/CacheAssertions.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/CacheAssertions.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/_test/CacheAssertions.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/DriverAssertions.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/DriverAssertions.java similarity index 90% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/DriverAssertions.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/_test/DriverAssertions.java index c89a96ba..b719524a 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/DriverAssertions.java +++ b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/DriverAssertions.java @@ -16,7 +16,6 @@ */ package _test; -import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.util.HasCache; import be.nbb.sdmx.facade.web.SdmxWebSource; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; @@ -49,11 +48,10 @@ public void assertDriverCompliance(SdmxWebDriver d) { assertThat(d.getName()).isNotBlank(); - assertThatNullPointerException().isThrownBy(() -> d.connect(null, ANY, context)); - assertThatNullPointerException().isThrownBy(() -> d.connect(validSource, null, context)); - assertThatNullPointerException().isThrownBy(() -> d.connect(validSource, ANY, null)); + assertThatNullPointerException().isThrownBy(() -> d.connect(null, context)); + assertThatNullPointerException().isThrownBy(() -> d.connect(validSource, null)); - assertThatIllegalArgumentException().isThrownBy(() -> d.connect(invalidSource, ANY, context)); + assertThatIllegalArgumentException().isThrownBy(() -> d.connect(invalidSource, context)); assertThat(d.getDefaultSources()).allSatisfy(o -> checkSource(o, d)); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/ConnectorsResource.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/ConnectorsResource.java similarity index 94% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/ConnectorsResource.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/ConnectorsResource.java index 3663ff61..1b49187c 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/ConnectorsResource.java +++ b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/ConnectorsResource.java @@ -23,7 +23,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.parser.ObsParser; +import be.nbb.sdmx.facade.parser.DataFactory; import internal.connectors.PortableTimeSeriesCursor; import internal.connectors.Connectors; import it.bancaditalia.oss.sdmx.api.DSDIdentifier; @@ -65,7 +65,7 @@ public SdmxRepository nbb() throws IOException { return SdmxRepository.builder() .structures(structs.stream().map(Connectors::toStructure).collect(Collectors.toList())) .flows(flows.stream().map(Connectors::toFlow).collect(Collectors.toList())) - .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) + .copyOf(ref, PortableTimeSeriesCursor.of(data, DataFactory.sdmx20(), Connectors.toStructure(structs.get(0)))) .name("NBB") .seriesKeysOnlySupported(false) .build(); @@ -84,7 +84,7 @@ public SdmxRepository ecb() throws IOException { return SdmxRepository.builder() .structures(structs.stream().map(Connectors::toStructure).collect(Collectors.toList())) .flows(flows.stream().map(Connectors::toFlow).collect(Collectors.toList())) - .copyOf(ref, new PortableTimeSeriesCursor(data, ObsParser.standard())) + .copyOf(ref, PortableTimeSeriesCursor.of(data, DataFactory.sdmx21(), Connectors.toStructure(structs.get(0)))) .name("ECB") .seriesKeysOnlySupported(true) .build(); diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/FacadeResource.java similarity index 94% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/FacadeResource.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/FacadeResource.java index 2f091a7b..f2513ab6 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/FacadeResource.java +++ b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/FacadeResource.java @@ -26,6 +26,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; @@ -104,7 +105,7 @@ private List flow20(ByteSource xml, LanguagePriorityList l) throws IOE } List data20(ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd, DataFactory.sdmx20()).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } @@ -118,7 +119,7 @@ private List flow21(ByteSource xml, LanguagePriorityList l) throws IOE } List data21(ByteSource xml, DataStructure dsd) throws IOException { - try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(xml::openReader)) { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd, DataFactory.sdmx21()).parseReader(xml::openReader)) { return SeriesSupport.copyOf(c); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/ParsersTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/ParsersTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/samples/ParsersTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/_test/samples/ParsersTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/ConnectorsTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/ConnectorsTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/ConnectorsTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java similarity index 83% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java index ac841b54..ba05dec0 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java +++ b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/PortableTimeSeriesCursorTest.java @@ -22,7 +22,7 @@ import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; import be.nbb.sdmx.facade.Obs; -import be.nbb.sdmx.facade.parser.ObsParser; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SeriesSupport; import it.bancaditalia.oss.sdmx.api.DataFlowStructure; import it.bancaditalia.oss.sdmx.api.PortableTimeSeries; @@ -40,18 +40,19 @@ */ public class PortableTimeSeriesCursorTest { + static DataFlowStructure DSD; static List> DATA; @BeforeClass public static void beforeClass() throws IOException { LanguagePriorityList l = LanguagePriorityList.parse("en"); - DataFlowStructure dsd = ConnectorsResource.struct21(SdmxSource.ECB_DATA_STRUCTURE, l).get(0); - DATA = ConnectorsResource.data21(SdmxSource.ECB_DATA, dsd, l); + DSD = ConnectorsResource.struct21(SdmxSource.ECB_DATA_STRUCTURE, l).get(0); + DATA = ConnectorsResource.data21(SdmxSource.ECB_DATA, DSD, l); } @Test public void test() throws IOException { - assertThat(SeriesSupport.asStream(() -> new PortableTimeSeriesCursor(DATA, ObsParser.standard()))) + assertThat(SeriesSupport.asStream(() -> PortableTimeSeriesCursor.of(DATA, DataFactory.sdmx21(), Connectors.toStructure(DSD)))) .hasSize(120) .allMatch(o -> o.getFreq().equals(Frequency.ANNUAL)) .element(0) @@ -70,6 +71,6 @@ public void test() throws IOException { @Test public void testCompliance() { - DataCursorAssert.assertCompliance(() -> new PortableTimeSeriesCursor(DATA, ObsParser.standard())); + DataCursorAssert.assertCompliance(() -> PortableTimeSeriesCursor.of(DATA, DataFactory.sdmx21(), Connectors.toStructure(DSD))); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/AbsDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/AbsDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/AbsDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/EurostatDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/EurostatDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/EurostatDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/IloDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/IloDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/IloDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/ImfDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/ImfDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/ImfDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/InseeDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/InseeDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/InseeDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/NbbDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/NbbDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/NbbDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/OecdDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/OecdDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/OecdDriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java similarity index 83% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java index f8536cc6..88061457 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java +++ b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/Sdmx20DriverTest.java @@ -16,7 +16,6 @@ */ package internal.connectors.drivers; -import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.web.SdmxWebSource; import java.io.IOException; import static org.assertj.core.api.Assertions.*; @@ -37,7 +36,7 @@ public void testCompliance() { @Test public void testConnect() throws IOException { - SdmxWebSource x = SdmxWebSource.builder().name("localhost").driver("sdmx20@connectors").endpointOf("http://localhost").build(); - assertThatCode(() -> new Sdmx20Driver().connect(x, ANY, SdmxWebContext.builder().build()).close()).doesNotThrowAnyException(); + SdmxWebSource x = SdmxWebSource.builder().name("localhost").driver("connectors:sdmx20").endpointOf("http://localhost").build(); + assertThatCode(() -> new Sdmx20Driver().connect(x, SdmxWebContext.builder().build()).close()).doesNotThrowAnyException(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/Sdmx21DriverTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java b/sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/UisDriverTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/connectors/drivers/UisDriverTest.java rename to sdmx-facade/sdmx-facade-connectors/src/test/java/internal/connectors/drivers/UisDriverTest.java diff --git a/sdmx-facade/sdmx-facade-file/pom.xml b/sdmx-facade/sdmx-facade-file/pom.xml index 5b86882b..1a381a4c 100644 --- a/sdmx-facade/sdmx-facade-file/pom.xml +++ b/sdmx-facade/sdmx-facade-file/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 sdmx-facade-file @@ -32,6 +32,10 @@ ${project.groupId} sdmx-facade-util + + ${project.groupId} + sdmx-facade-util-xml + junit diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java index 1033b8d8..14c107a2 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/be/nbb/sdmx/facade/file/SdmxFileConnection.java @@ -17,13 +17,15 @@ package be.nbb.sdmx.facade.file; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; import java.io.IOException; +import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -52,12 +54,17 @@ default DataStructure getStructure() throws IOException { } @Nonnull - default DataCursor getCursor(@Nonnull DataQuery query) throws IOException { - return getCursor(getDataflowRef(), query); + default List getData(@Nonnull Key key, @Nonnull DataFilter filter) throws IOException { + return getData(getDataflowRef(), key, filter); } @Nonnull - default Stream getStream(@Nonnull DataQuery query) throws IOException { - return getStream(getDataflowRef(), query); + default Stream getDataStream(@Nonnull Key key, @Nonnull DataFilter filter) throws IOException { + return getDataStream(getDataflowRef(), key, filter); + } + + @Nonnull + default DataCursor getDataCursor(@Nonnull Key key, @Nonnull DataFilter filter) throws IOException { + return getDataCursor(getDataflowRef(), key, filter); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java index 6ebdc92e..e1fe872f 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/SdmxFileConnectionImpl.java @@ -21,16 +21,16 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.DataQueryDetail; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.file.SdmxFileConnection; import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; +import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.stream.Stream; /** @@ -58,7 +58,7 @@ public DataflowRef getDataflowRef() throws IOException { } @Override - public Set getFlows() throws IOException { + public Collection getFlows() throws IOException { checkState(); return Collections.singleton(dataflow); } @@ -90,28 +90,40 @@ public DataStructure getStructure(DataflowRef flowRef) throws IOException { } @Override - public DataCursor getCursor(DataQuery query) throws IOException { - checkState(); - Objects.requireNonNull(query); - return resource.loadData(resource.decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); + public List getData(Key key, DataFilter filter) throws IOException { + return SeriesSupport.asList(() -> getDataCursor(key, filter)); } @Override - public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { - checkState(); - checkFlowRef(flowRef); - Objects.requireNonNull(query); - return resource.loadData(resource.decode(), dataflow.getRef(), query.getKey(), query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)); + public List getData(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + return SeriesSupport.asList(() -> getDataCursor(flowRef, key, filter)); + } + + @Override + public Stream getDataStream(Key key, DataFilter filter) throws IOException { + return SeriesSupport.asStream(() -> getDataCursor(key, filter)); } @Override - public Stream getStream(DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getCursor(query)); + public Stream getDataStream(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + return SeriesSupport.asStream(() -> getDataCursor(flowRef, key, filter)); } @Override - public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getCursor(flowRef, query)); + public DataCursor getDataCursor(Key key, DataFilter filter) throws IOException { + checkState(); + Objects.requireNonNull(key); + Objects.requireNonNull(filter); + return resource.loadData(resource.decode(), dataflow.getRef(), key, filter.isSeriesKeyOnly()); + } + + @Override + public DataCursor getDataCursor(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + checkState(); + checkFlowRef(flowRef); + Objects.requireNonNull(key); + Objects.requireNonNull(filter); + return resource.loadData(resource.decode(), dataflow.getRef(), key, filter.isSeriesKeyOnly()); } @Override @@ -126,7 +138,7 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw SdmxExceptions.connectionClosed(); + throw SdmxExceptions.connectionClosed("fixme"); } } diff --git a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java index 66c762ba..010f5950 100644 --- a/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java +++ b/sdmx-facade/sdmx-facade-file/src/main/java/internal/file/xml/DataTypeProbe.java @@ -16,8 +16,7 @@ */ package internal.file.xml; -import be.nbb.sdmx.facade.xml.Sdmxml; -import static be.nbb.sdmx.facade.xml.Sdmxml.*; +import static be.nbb.sdmx.facade.xml.SdmxmlUri.*; import be.nbb.util.StaxUtil; import static internal.file.SdmxDecoder.DataType.*; import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; @@ -51,9 +50,9 @@ private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws level++; if (level == 2 && reader.getLocalName().equals("Header")) { URI uri = URI.create(reader.getNamespaceURI()); - if (Sdmxml.equals(NS_V10_URI, uri)) { + if (NS_V10_URI.is(uri)) { return UNKNOWN; - } else if (Sdmxml.equals(NS_V20_URI, uri)) { + } else if (NS_V20_URI.is(uri)) { while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: @@ -68,7 +67,7 @@ private static SdmxDecoder.DataType probeDataType(XMLStreamReader reader) throws } } return COMPACT20; - } else if (Sdmxml.equals(NS_V21_URI, uri)) { + } else if (NS_V21_URI.is(uri)) { while (reader.hasNext()) { switch (reader.next()) { case START_ELEMENT: diff --git a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java index 946b50d4..493a0a2c 100644 --- a/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-file/src/test/java/internal/file/SdmxFileConnectionImplTest.java @@ -20,7 +20,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Frequency; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; @@ -58,10 +58,10 @@ public void testFile() throws IOException { assertThat(conn.getDataflowRef()).isEqualTo(files.asDataflowRef()); assertThat(conn.getFlow()).isEqualTo(conn.getFlow(files.asDataflowRef())); assertThat(conn.getStructure()).isEqualTo(conn.getStructure(files.asDataflowRef())); - assertThatNullPointerException().isThrownBy(() -> conn.getCursor(null)); - assertThatNullPointerException().isThrownBy(() -> conn.getStream(null)); - try (Stream stream = conn.getStream(DataQuery.of(Key.ALL, false))) { - assertThat(stream).containsExactly(conn.getStream(DataQuery.of(Key.ALL, false)).toArray(Series[]::new)); + assertThatNullPointerException().isThrownBy(() -> conn.getDataCursor(Key.ALL, null)); + assertThatNullPointerException().isThrownBy(() -> conn.getDataStream(Key.ALL, null)); + try (Stream stream = conn.getDataStream(Key.ALL, DataFilter.ALL)) { + assertThat(stream).containsExactly(conn.getDataStream(Key.ALL, DataFilter.ALL).toArray(Series[]::new)); } } @@ -80,7 +80,7 @@ public void testCompactData21() throws IOException { Key key = Key.of("A", "BEL", "1", "0", "0", "0", "OVGD"); - try (DataCursor o = conn.getCursor(files.asDataflowRef(), DataQuery.of(Key.ALL, false))) { + try (DataCursor o = conn.getDataCursor(files.asDataflowRef(), Key.ALL, DataFilter.ALL)) { assertThat(o.nextSeries()).isTrue(); assertThat(o.getSeriesKey()).isEqualTo(key); assertThat(o.getSeriesFrequency()).isEqualTo(Frequency.ANNUAL); diff --git a/sdmx-facade/sdmx-facade-samples/pom.xml b/sdmx-facade/sdmx-facade-samples/pom.xml index 9547000c..14832ae8 100644 --- a/sdmx-facade/sdmx-facade-samples/pom.xml +++ b/sdmx-facade/sdmx-facade-samples/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 sdmx-facade-samples diff --git a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java index 1b2091c4..e2851dfc 100644 --- a/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java +++ b/sdmx-facade/sdmx-facade-samples/src/main/java/be/nbb/sdmx/facade/tck/ConnectionAssert.java @@ -19,7 +19,7 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.Series; @@ -49,8 +49,9 @@ public static void assertCompliance(Callable supplier, DataflowR public static void assertCompliance(SoftAssertions s, Callable supplier, DataflowRef ref) throws Exception { try (SdmxConnection conn = supplier.call()) { assertNonnull(s, conn, ref); - DataCursorAssert.assertCompliance(s, () -> conn.getCursor(ref, ALL)); - s.assertThat(conn.getStream(ref, ALL)).containsExactlyElementsOf(cursorToSeries(ref, ALL, conn)); + DataCursorAssert.assertCompliance(s, () -> conn.getDataCursor(ref, Key.ALL, DataFilter.ALL)); + s.assertThat(conn.getData(ref, Key.ALL, DataFilter.ALL)).containsExactlyElementsOf(cursorToSeries(ref, Key.ALL, DataFilter.ALL, conn)); + s.assertThat(conn.getDataStream(ref, Key.ALL, DataFilter.ALL)).containsExactlyElementsOf(cursorToSeries(ref, Key.ALL, DataFilter.ALL, conn)); s.assertThat(conn.getFlows()).isNotEmpty().filteredOn(ref::containsRef).isNotEmpty(); s.assertThat(conn.getFlow(ref)).isNotNull(); s.assertThat(conn.getStructure(ref)).isNotNull(); @@ -62,8 +63,9 @@ public static void assertCompliance(SoftAssertions s, Callable s s.fail("Subsequent calls to #close must not raise exception", ex); } - assertState(s, supplier, o -> o.getCursor(ref, ALL), "getCursor(DataflowRef, DataQuery)"); - assertState(s, supplier, o -> o.getStream(ref, ALL), "getStream(DataflowRef, DataQuery)"); + assertState(s, supplier, o -> o.getData(ref, Key.ALL, DataFilter.ALL), "getData(DataflowRef, Key, DataFilter)"); + assertState(s, supplier, o -> o.getDataStream(ref, Key.ALL, DataFilter.ALL), "getDataStream(DataflowRef, Key, DataFilter)"); + assertState(s, supplier, o -> o.getDataCursor(ref, Key.ALL, DataFilter.ALL), "getDataCursor(DataflowRef, Key, DataFilter)"); assertState(s, supplier, o -> o.getStructure(ref), "getStructure(DataflowRef)"); assertState(s, supplier, o -> o.getFlow(ref), "getFlow(DataflowRef)"); assertState(s, supplier, SdmxConnection::getFlows, "getFlows()"); @@ -71,20 +73,44 @@ public static void assertCompliance(SoftAssertions s, Callable s @SuppressWarnings("null") private static void assertNonnull(SoftAssertions s, SdmxConnection conn, DataflowRef ref) { - s.assertThatThrownBy(() -> conn.getCursor(null, ALL)) - .as("Expecting 'getCursor(DataflowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getDataCursor(null, Key.ALL, DataFilter.ALL)) + .as("Expecting 'getDataCursor(DataflowRef, Key, DataFilter)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getCursor(ref, null)) - .as("Expecting 'getCursor(DataflowRef, DataQuery)' to raise NPE when called with null query") + s.assertThatThrownBy(() -> conn.getData(null, Key.ALL, DataFilter.ALL)) + .as("Expecting 'getData(DataflowRef, Key, DataFilter)' to raise NPE when called with null flowRef") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getStream(null, ALL)) - .as("Expecting 'getStream(DataflowRef, DataQuery)' to raise NPE when called with null flowRef") + s.assertThatThrownBy(() -> conn.getData(ref, null, DataFilter.ALL)) + .as("Expecting 'getData(DataflowRef, Key, DataFilter)' to raise NPE when called with null key") .isInstanceOf(NullPointerException.class); - s.assertThatThrownBy(() -> conn.getStream(ref, null)) - .as("Expecting 'getStream(DataflowRef, DataQuery)' to raise NPE when called with null query") + s.assertThatThrownBy(() -> conn.getData(ref, Key.ALL, null)) + .as("Expecting 'getData(DataflowRef, Key, DataFilter)' to raise NPE when called with null query") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataStream(null, Key.ALL, DataFilter.ALL)) + .as("Expecting 'getDataStream(DataflowRef, Key, DataFilter)' to raise NPE when called with null flowRef") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataStream(ref, null, DataFilter.ALL)) + .as("Expecting 'getDataStream(DataflowRef, Key, DataFilter)' to raise NPE when called with null key") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataStream(ref, Key.ALL, null)) + .as("Expecting 'getDataStream(DataflowRef, Key, DataFilter)' to raise NPE when called with null query") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataCursor(null, Key.ALL, DataFilter.ALL)) + .as("Expecting 'getDataCursor(DataflowRef, Key, DataFilter)' to raise NPE when called with null flowRef") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataCursor(ref, null, DataFilter.ALL)) + .as("Expecting 'getDataCursor(DataflowRef, Key, DataFilter)' to raise NPE when called with null key") + .isInstanceOf(NullPointerException.class); + + s.assertThatThrownBy(() -> conn.getDataCursor(ref, Key.ALL, null)) + .as("Expecting 'getDataCursor(DataflowRef, Key, DataFilter)' to raise NPE when called with null query") .isInstanceOf(NullPointerException.class); s.assertThatThrownBy(() -> conn.getStructure(null)) @@ -106,9 +132,9 @@ private static void assertState(SoftAssertions s, Callable suppl } } - private static List cursorToSeries(DataflowRef ref, DataQuery query, SdmxConnection conn) throws IOException { + private static List cursorToSeries(DataflowRef ref, Key key, DataFilter filter, SdmxConnection conn) throws IOException { List result = new ArrayList(); - try (DataCursor c = conn.getCursor(ref, query)) { + try (DataCursor c = conn.getDataCursor(ref, key, filter)) { while (c.nextSeries()) { Series.Builder series = Series.builder(); series.key(c.getSeriesKey()); @@ -122,6 +148,4 @@ private static List cursorToSeries(DataflowRef ref, DataQuery query, Sdm } return result; } - - private static final DataQuery ALL = DataQuery.of(Key.ALL, false); } diff --git a/sdmx-facade/sdmx-facade-util-web/pom.xml b/sdmx-facade/sdmx-facade-util-web/pom.xml new file mode 100644 index 00000000..f7dde999 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + + be.nbb.sdmx + sdmx-facade-parent + 2.2.3 + + + sdmx-facade-util-web + jar + + + + + com.google.code.findbugs + jsr305 + provided + + + org.projectlombok + lombok + provided + + + org.netbeans.api + org-openide-util-lookup + provided + + + + ${project.groupId} + sdmx-facade-api + + + ${project.groupId} + sdmx-facade-util + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + ${project.groupId} + sdmx-facade-samples + test + + + ${project.groupId} + sdmx-facade-util-xml + test + + + \ No newline at end of file diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/util/drivers/InseeDialect.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/InseeDialect.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/util/drivers/InseeDialect.java diff --git a/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/util/drivers/SdmxWebResource.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/util/drivers/SdmxWebResource.java new file mode 100644 index 00000000..5fc8f41b --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/util/drivers/SdmxWebResource.java @@ -0,0 +1,85 @@ +/* + * Copyright 2018 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util.drivers; + +import be.nbb.sdmx.facade.web.SdmxWebSource; +import ioutil.Stax; +import ioutil.Xml; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public final class SdmxWebResource { + + public static List load(String resource) { + try { + return getParser().parseStream(() -> SdmxWebResource.class.getResourceAsStream(resource)); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + static Xml.Parser> getParser() throws IOException { + return Stax.StreamParser.valueOf(SdmxWebResource::parse); + } + + private static List parse(XMLStreamReader reader) throws XMLStreamException { + List result = new ArrayList<>(); + SdmxWebSource.Builder item = SdmxWebSource.builder(); + while (reader.hasNext()) { + switch (reader.next()) { + case XMLStreamReader.START_ELEMENT: + switch (reader.getLocalName()) { + case "source": + item = SdmxWebSource.builder(); + break; + case "name": + item.name(reader.getElementText()); + break; + case "description": + item.description(reader.getElementText()); + break; + case "driver": + item.driver(reader.getElementText()); + break; + case "endpoint": + item.endpointOf(reader.getElementText()); + break; + case "property": + item.property(reader.getAttributeValue(null, "key"), reader.getAttributeValue(null, "value")); + break; + } + break; + case XMLStreamReader.END_ELEMENT: + switch (reader.getLocalName()) { + case "source": + result.add(item.build()); + break; + } + break; + } + } + return result; + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/CachedWebClient.java similarity index 82% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedWebClient.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/CachedWebClient.java index 95a4ae54..c5492839 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/CachedWebClient.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/CachedWebClient.java @@ -18,8 +18,6 @@ import be.nbb.sdmx.facade.DataCursor; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataQueryDetail; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; @@ -60,6 +58,11 @@ static CachedWebClient of(SdmxWebClient delegate, String base, ConcurrentMap cac this.idOfKeysOnly = TypedId.of("keys://" + base); } + @Override + public String getName() throws IOException { + return delegate.getName(); + } + @Override public List getFlows() throws IOException { return loadDataFlowsWithCache(); @@ -77,11 +80,12 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - if (!query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY)) { - return delegate.getData(flowRef, query, dsd); + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + if (!request.getFilter().isSeriesKeyOnly()) { + return delegate.getData(request, dsd); } - return loadKeysOnlyWithCache(flowRef, dsd, query).getCursor(flowRef, query) + return loadKeysOnlyWithCache(request, dsd) + .getDataCursor(request.getFlowRef(), request.getKey(), request.getFilter()) .orElseThrow(() -> new IOException("Data not found")); } @@ -119,11 +123,11 @@ private DataStructure loadDataStructureWithCache(DataStructureRef ref) throws IO return result; } - private SdmxRepository loadKeysOnlyWithCache(DataflowRef flowRef, DataStructure dsd, DataQuery query) throws IOException { - TypedId id = idOfKeysOnly.with(flowRef); + private SdmxRepository loadKeysOnlyWithCache(DataRequest request, DataStructure dsd) throws IOException { + TypedId id = idOfKeysOnly.with(request.getFlowRef()); SdmxRepository result = cache.get(id); - if (result == null || isBroaderRequest(query.getKey(), result)) { - result = copyDataKeys(flowRef, dsd, query); + if (result == null || isBroaderRequest(request.getKey(), result)) { + result = copyDataKeys(request, dsd); cache.put(id, result); } return result; @@ -158,11 +162,11 @@ private boolean isBroaderRequest(Key key, SdmxRepository repo) { return key.supersedes(Key.parse(repo.getName())); } - private SdmxRepository copyDataKeys(DataflowRef flowRef, DataStructure structure, DataQuery query) throws IOException { - try (DataCursor cursor = delegate.getData(flowRef, query, structure)) { + private SdmxRepository copyDataKeys(DataRequest request, DataStructure structure) throws IOException { + try (DataCursor cursor = delegate.getData(request, structure)) { return SdmxRepository.builder() - .copyOf(flowRef, cursor) - .name(query.getKey().toString()) + .copyOf(request.getFlowRef(), cursor) + .name(request.getKey().toString()) .build(); } } diff --git a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/DataRequest.java similarity index 69% rename from sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/DataRequest.java index a460c90e..5a815c69 100644 --- a/sdmx-facade/sdmx-facade-api/src/main/java/be/nbb/sdmx/facade/DataQueryDetail.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/DataRequest.java @@ -14,13 +14,25 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ -package be.nbb.sdmx.facade; +package internal.web; + +import be.nbb.sdmx.facade.DataFilter; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.Key; /** * * @author Philippe Charles */ -public enum DataQueryDetail { +@lombok.Value +public class DataRequest { + + @lombok.NonNull + private DataflowRef flowRef; + + @lombok.NonNull + private Key key; - FULL, SERIES_KEYS_ONLY + @lombok.NonNull + private DataFilter filter; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/FailsafeWebClient.java similarity index 69% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeWebClient.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/FailsafeWebClient.java index ec6a2514..0f6f4baa 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/FailsafeWebClient.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/FailsafeWebClient.java @@ -17,7 +17,6 @@ package internal.web; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; @@ -27,18 +26,38 @@ import java.time.Duration; import java.util.List; import java.util.logging.Level; +import java.util.logging.Logger; /** * * @author Philippe Charles */ @lombok.AllArgsConstructor(staticName = "of") -@lombok.extern.java.Log final class FailsafeWebClient implements SdmxWebClient { @lombok.NonNull private final SdmxWebClient delegate; + @lombok.NonNull + private final Logger logger; + + @Override + public String getName() throws IOException { + String result; + + try { + result = delegate.getName(); + } catch (RuntimeException ex) { + throw unexpected(ex, "Unexpected exception while getting name"); + } + + if (result == null) { + throw unexpectedNull("Unexpected null while getting name"); + } + + return result; + } + @Override public List getFlows() throws IOException { List result; @@ -46,11 +65,11 @@ public List getFlows() throws IOException { try { result = delegate.getFlows(); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting datasets"); + throw unexpected(ex, "Unexpected exception while getting datasets with '%s'", getId()); } if (result == null) { - throw unexpectedNull("Unexpected null while getting datasets"); + throw unexpectedNull("Unexpected null while getting datasets with '%s'", getId()); } return result; @@ -63,11 +82,11 @@ public Dataflow getFlow(DataflowRef ref) throws IOException { try { result = delegate.getFlow(ref); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting details from dataset '%s'", ref); + throw unexpected(ex, "Unexpected exception while getting details for dataset '%s' with '%s'", ref, getId()); } if (result == null) { - throw unexpectedNull("Unexpected null while getting details from dataset '%s'", ref); + throw unexpectedNull("Unexpected null while getting details for dataset '%s' with '%s'", ref, getId()); } return result; @@ -80,28 +99,28 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { try { result = delegate.getStructure(ref); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting data structure from dataset '%s'", ref); + throw unexpected(ex, "Unexpected exception while getting data structure for dataset '%s' with '%s'", ref, getId()); } if (result == null) { - throw unexpectedNull("Unexpected null while getting data structure from dataset '%s'", ref); + throw unexpectedNull("Unexpected null while getting data structure for dataset '%s' with '%s'", ref, getId()); } return result; } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { DataCursor result; try { - result = delegate.getData(flowRef, query, dsd); + result = delegate.getData(request, dsd); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while getting data from dataset '%s' with key '%s'", flowRef, query.getKey()); + throw unexpected(ex, "Unexpected exception while getting data '%s' with '%s'", request, getId()); } if (result == null) { - throw unexpectedNull("Unexpected null while getting data from dataset '%s' with key '%s'", flowRef, query.getKey()); + throw unexpectedNull("Unexpected null while getting data '%s' with '%s'", request, getId()); } return result; @@ -112,7 +131,7 @@ public boolean isSeriesKeysOnlySupported() throws IOException { try { return delegate.isSeriesKeysOnlySupported(); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while checking keys-only support"); + throw unexpected(ex, "Unexpected exception while checking keys-only support with '%s'", getId()); } } @@ -121,7 +140,7 @@ public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException try { return delegate.peekStructureRef(flowRef); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while peeking struct ref for dataset '%s'", flowRef); + throw unexpected(ex, "Unexpected exception while peeking struct ref for dataset '%s' with '%s'", flowRef, getId()); } } @@ -132,24 +151,34 @@ public Duration ping() throws IOException { try { result = delegate.ping(); } catch (RuntimeException ex) { - throw unexpected(ex, "Unexpected exception while pinging resource"); + throw unexpected(ex, "Unexpected exception while pinging resource with '%s'", getId()); } if (result == null) { - throw unexpectedNull("Unexpected null while pinging resource"); + throw unexpectedNull("Unexpected null while pinging resource with '%s'", getId()); } return result; } - private static IOException unexpected(RuntimeException ex, String format, Object... args) { - log.log(Level.WARNING, String.format(format, args)); + @SuppressWarnings("null") + private String getId() { + try { + String result = delegate.getName(); + return result != null && !result.isEmpty() ? result : "NULL"; + } catch (Exception ex) { + return ex.getMessage(); + } + } + + private IOException unexpected(RuntimeException ex, String format, Object... args) { + logger.log(Level.WARNING, String.format(format, args), ex); return new UnexpectedIOException(ex); } - private static IOException unexpectedNull(String format, Object... args) { + private IOException unexpectedNull(String format, Object... args) { String msg = String.format(format, args); - log.log(Level.WARNING, msg); + logger.log(Level.WARNING, msg); return new UnexpectedIOException(new NullPointerException(msg)); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebClient.java similarity index 82% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebClient.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebClient.java index 03780af8..1c2a509f 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebClient.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebClient.java @@ -17,12 +17,10 @@ package internal.web; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.web.SdmxWebSource; import java.io.IOException; import java.time.Duration; @@ -37,6 +35,9 @@ */ public interface SdmxWebClient { + @Nonnull + String getName() throws IOException; + @Nonnull List getFlows() throws IOException; @@ -47,12 +48,12 @@ public interface SdmxWebClient { DataStructure getStructure(@Nonnull DataStructureRef ref) throws IOException; @Nonnull - DataCursor getData(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query, @Nonnull DataStructure dsd) throws IOException; + DataCursor getData(@Nonnull DataRequest request, @Nonnull DataStructure dsd) throws IOException; boolean isSeriesKeysOnlySupported() throws IOException; @Nullable - DataStructureRef peekStructureRef(@Nonnull DataflowRef flowRef) throws IOException; + DataStructureRef peekStructureRef(@Nonnull DataflowRef ref) throws IOException; @Nonnull Duration ping() throws IOException; @@ -63,8 +64,12 @@ interface Supplier { @Nonnull SdmxWebClient get( @Nonnull SdmxWebSource source, - @Nonnull LanguagePriorityList langs, @Nonnull SdmxWebContext context ); } + + @Nonnull + static String getClientName(@Nonnull SdmxWebSource source) { + return source.getDriver() + ":" + source.getName(); + } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebConnectionImpl.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebConnectionImpl.java similarity index 71% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebConnectionImpl.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebConnectionImpl.java index 262fb26e..ee69dab3 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebConnectionImpl.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebConnectionImpl.java @@ -20,17 +20,17 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.DataQueryDetail; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.Series; import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import be.nbb.sdmx.facade.web.SdmxWebConnection; import java.io.IOException; import java.time.Duration; -import java.util.HashSet; -import java.util.Set; +import java.util.Collection; +import java.util.List; import java.util.stream.Stream; /** @@ -42,12 +42,16 @@ final class SdmxWebConnectionImpl implements SdmxWebConnection { @lombok.NonNull private final SdmxWebClient client; + + @lombok.NonNull + private final String driver; + private boolean closed = false; @Override - public Set getFlows() throws IOException { + public Collection getFlows() throws IOException { checkState(); - return new HashSet<>(client.getFlows()); + return client.getFlows(); } @Override @@ -70,9 +74,19 @@ public DataStructure getStructure(DataflowRef flowRef) throws IOException { } @Override - public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { + public List getData(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + return SeriesSupport.asList(() -> getDataCursor(flowRef, key, filter)); + } + + @Override + public Stream getDataStream(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + return SeriesSupport.asStream(() -> getDataCursor(flowRef, key, filter)); + } + + @Override + public DataCursor getDataCursor(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { checkState(); - checkQuery(query); + checkQuery(filter); DataStructureRef structRef = client.peekStructureRef(flowRef); if (structRef == null) { @@ -82,12 +96,7 @@ public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOExcep } DataStructure structure = client.getStructure(structRef); - return client.getData(flowRef, query, structure); - } - - @Override - public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { - return SeriesSupport.asStream(() -> getCursor(flowRef, query)); + return client.getData(new DataRequest(flowRef, key, filter), structure); } @Override @@ -101,6 +110,12 @@ public Duration ping() throws IOException { return client.ping(); } + @Override + public String getDriver() throws IOException { + checkState(); + return driver; + } + @Override public void close() throws IOException { closed = true; @@ -108,13 +123,12 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw SdmxExceptions.connectionClosed(); + throw SdmxExceptions.connectionClosed(client.getName()); } } - private void checkQuery(DataQuery query) throws IOException { - boolean serieskeysonly = query.getDetail().equals(DataQueryDetail.SERIES_KEYS_ONLY); - if (serieskeysonly && !isSeriesKeysOnlySupported()) { + private void checkQuery(DataFilter filter) throws IOException { + if (filter.isSeriesKeyOnly() && !isSeriesKeysOnlySupported()) { throw new IllegalStateException("serieskeysonly not supported"); } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebDriverSupport.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebDriverSupport.java similarity index 81% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebDriverSupport.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebDriverSupport.java index 8d42507d..2dd48798 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebDriverSupport.java +++ b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebDriverSupport.java @@ -39,9 +39,13 @@ @ThreadSafe public final class SdmxWebDriverSupport implements SdmxWebDriver, HasCache { + @lombok.Getter @lombok.NonNull private final String name; + @lombok.Getter + private final int rank; + @lombok.NonNull private final SdmxWebClient.Supplier client; @@ -58,21 +62,15 @@ public final class SdmxWebDriverSupport implements SdmxWebDriver, HasCache { private final HasCache cacheSupport = HasCache.of(ConcurrentHashMap::new); @Override - public String getName() { - return name; - } - - @Override - public SdmxWebConnection connect(SdmxWebSource source, LanguagePriorityList langs, SdmxWebContext context) throws IOException { + public SdmxWebConnection connect(SdmxWebSource source, SdmxWebContext context) throws IOException { Objects.requireNonNull(source); - Objects.requireNonNull(langs); Objects.requireNonNull(context); if (!source.getDriver().equals(name)) { throw new IllegalArgumentException(source.toString()); } - return SdmxWebConnectionImpl.of(getClient(source, langs, context)); + return SdmxWebConnectionImpl.of(getClient(source, context), name); } @Override @@ -95,13 +93,13 @@ public void setCache(ConcurrentMap cache) { cacheSupport.setCache(cache); } - private SdmxWebClient getClient(SdmxWebSource source, LanguagePriorityList langs, SdmxWebContext context) throws IOException { - SdmxWebClient origin = client.get(source, langs, context); - SdmxWebClient cached = CachedWebClient.of(origin, getBase(source, langs), getCache(), clock, SdmxWebProperty.getCacheTtl(source.getProperties())); - return FailsafeWebClient.of(cached); + private SdmxWebClient getClient(SdmxWebSource source, SdmxWebContext context) throws IOException { + SdmxWebClient origin = client.get(source, context); + SdmxWebClient cached = CachedWebClient.of(origin, getBase(source, context.getLanguages()), getCache(), clock, SdmxWebProperty.getCacheTtl(source.getProperties())); + return FailsafeWebClient.of(cached, context.getLogger()); } - private static String getBase(SdmxWebSource source, LanguagePriorityList languages) throws IOException { + private static String getBase(SdmxWebSource source, LanguagePriorityList languages) { return source.getEndpoint().getHost() + languages.toString() + "/"; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebProperty.java b/sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebProperty.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/SdmxWebProperty.java rename to sdmx-facade/sdmx-facade-util-web/src/main/java/internal/web/SdmxWebProperty.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/DialectAssertions.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/DialectAssertions.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/DialectAssertions.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/_test/DialectAssertions.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/CallStackWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XCallStackWebClient.java similarity index 86% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/client/CallStackWebClient.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XCallStackWebClient.java index d8ff1e9a..38ba5d4e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/CallStackWebClient.java +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XCallStackWebClient.java @@ -17,11 +17,11 @@ package _test.client; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; +import internal.web.DataRequest; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -33,7 +33,7 @@ * @author Philippe Charles */ @lombok.RequiredArgsConstructor(staticName = "of") -public final class CallStackWebClient implements SdmxWebClient { +public final class XCallStackWebClient implements SdmxWebClient { @lombok.NonNull private final SdmxWebClient delegate; @@ -41,6 +41,11 @@ public final class CallStackWebClient implements SdmxWebClient { @lombok.NonNull private final AtomicInteger count; + @Override + public String getName() throws IOException { + return delegate.getName(); + } + @Override public List getFlows() throws IOException { count.incrementAndGet(); @@ -60,9 +65,9 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { count.incrementAndGet(); - return delegate.getData(flowRef, query, dsd); + return delegate.getData(request, dsd); } @Override diff --git a/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XFailingWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XFailingWebClient.java new file mode 100644 index 00000000..02c151d4 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XFailingWebClient.java @@ -0,0 +1,171 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package _test.client; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import internal.web.DataRequest; +import internal.web.SdmxWebClient; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +/** + * + * @author Philippe Charles + */ +public enum XFailingWebClient implements SdmxWebClient { + + EXPECTED { + @Override + public String getName() throws IOException { + throw new CustomIOException(); + } + + @Override + public List getFlows() throws IOException { + throw new CustomIOException(); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + throw new CustomIOException(); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + throw new CustomIOException(); + } + + @Override + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + throw new CustomIOException(); + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + throw new CustomIOException(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + throw new CustomIOException(); + } + + @Override + public Duration ping() throws IOException { + throw new CustomIOException(); + } + }, + UNEXPECTED { + @Override + public String getName() throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public List getFlows() throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + throw new CustomRuntimeException(); + } + + @Override + public Duration ping() throws IOException { + throw new CustomRuntimeException(); + } + }, + NULL { + @Override + public String getName() throws IOException { + return null; + } + + @Override + public List getFlows() throws IOException { + return null; + } + + @Override + public Dataflow getFlow(DataflowRef ref) throws IOException { + Objects.requireNonNull(ref); + return null; + } + + @Override + public DataStructure getStructure(DataStructureRef ref) throws IOException { + Objects.requireNonNull(ref); + return null; + } + + @Override + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + Objects.requireNonNull(request); + Objects.requireNonNull(dsd); + return null; + } + + @Override + public boolean isSeriesKeysOnlySupported() throws IOException { + return false; + } + + @Override + public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { + Objects.requireNonNull(flowRef); + return null; + } + + @Override + public Duration ping() throws IOException { + return null; + } + }; + + private static final class CustomIOException extends IOException { + } + + private static final class CustomRuntimeException extends RuntimeException { + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/RepoWebClient.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XRepoWebClient.java similarity index 80% rename from sdmx-facade/sdmx-facade-web/src/test/java/_test/client/RepoWebClient.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XRepoWebClient.java index 28998c71..f44136bc 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/RepoWebClient.java +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/client/XRepoWebClient.java @@ -17,13 +17,13 @@ package _test.client; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.repo.SdmxRepository; import be.nbb.sdmx.facade.util.SdmxExceptions; +import internal.web.DataRequest; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -35,11 +35,16 @@ * @author Philippe Charles */ @lombok.AllArgsConstructor(staticName = "of") -public final class RepoWebClient implements SdmxWebClient { +public final class XRepoWebClient implements SdmxWebClient { @lombok.NonNull private final SdmxRepository repo; + @Override + public String getName() throws IOException { + return repo.getName(); + } + @Override public List getFlows() throws IOException { return new ArrayList(repo.getFlows()); @@ -48,19 +53,19 @@ public List getFlows() throws IOException { @Override public Dataflow getFlow(DataflowRef ref) throws IOException { return repo.getFlow(ref) - .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); + .orElseThrow(() -> SdmxExceptions.missingFlow(repo.getName(), ref)); } @Override public DataStructure getStructure(DataStructureRef ref) throws IOException { return repo.getStructure(ref) - .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); + .orElseThrow(() -> SdmxExceptions.missingStructure(repo.getName(), ref)); } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - return repo.getCursor(flowRef, query) - .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + return repo.getDataCursor(request.getFlowRef(), request.getKey(), request.getFilter()) + .orElseThrow(() -> SdmxExceptions.missingData(repo.getName(), request.getFlowRef())); } @Override diff --git a/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/samples/FacadeResource.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/samples/FacadeResource.java new file mode 100644 index 00000000..f2513ab6 --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/_test/samples/FacadeResource.java @@ -0,0 +1,131 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package _test.samples; + +import be.nbb.sdmx.facade.DataCursor; +import be.nbb.sdmx.facade.DataStructure; +import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Dataflow; +import be.nbb.sdmx.facade.DataflowRef; +import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.samples.ByteSource; +import be.nbb.sdmx.facade.samples.SdmxSource; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.Series; +import be.nbb.sdmx.facade.parser.DataFactory; +import be.nbb.sdmx.facade.util.SeriesSupport; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class FacadeResource { + + public static final DataflowRef ECB_FLOW_REF = DataflowRef.of("ECB", "AME", "1.0"); + public static final DataStructureRef ECB_STRUCT_REF = DataStructureRef.of("ECB", "ECB_AME1", "1.0"); + + public static final DataflowRef NBB_FLOW_REF = DataflowRef.of("NBB", "TEST_DATASET", null); + + public SdmxRepository nbb() throws IOException { + SdmxRepository result = NBB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct20(SdmxSource.NBB_DATA_STRUCTURE, l); + List flows = flow20(SdmxSource.NBB_DATA_STRUCTURE, l); + List data = data20(SdmxSource.NBB_DATA, structs.get(0)); + + result = SdmxRepository.builder() + .structures(structs) + .flows(flows) + .data(NBB_FLOW_REF, data) + .name("NBB") + .seriesKeysOnlySupported(false) + .build(); + + NBB.set(result); + } + return result; + } + + public SdmxRepository ecb() throws IOException { + SdmxRepository result = ECB.get(); + if (result == null) { + LanguagePriorityList l = LanguagePriorityList.parse("fr"); + + List structs = struct21(SdmxSource.ECB_DATA_STRUCTURE, l); + List flows = flow21(SdmxSource.ECB_DATAFLOWS, l); + List data = data21(SdmxSource.ECB_DATA, structs.get(0)); + + result = SdmxRepository.builder() + .structures(structs) + .flows(flows) + .data(ECB_FLOW_REF, data) + .name("ECB") + .seriesKeysOnlySupported(true) + .build(); + + ECB.set(result); + } + return result; + } + + private static final AtomicReference NBB = new AtomicReference<>(); + private static final AtomicReference ECB = new AtomicReference<>(); + + private List struct20(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct20(l).parseReader(xml::openReader); + } + + private List flow20(ByteSource xml, LanguagePriorityList l) throws IOException { + // FIXME: find sample of dataflow20 ? + return struct20(xml, l).stream() + .map(FacadeResource::asDataflow) + .collect(Collectors.toList()); + } + + List data20(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData20(dsd, DataFactory.sdmx20()).parseReader(xml::openReader)) { + return SeriesSupport.copyOf(c); + } + } + + private List struct21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.struct21(l).parseReader(xml::openReader); + } + + private List flow21(ByteSource xml, LanguagePriorityList l) throws IOException { + return SdmxXmlStreams.flow21(l).parseReader(xml::openReader); + } + + List data21(ByteSource xml, DataStructure dsd) throws IOException { + try (DataCursor c = SdmxXmlStreams.genericData21(dsd, DataFactory.sdmx21()).parseReader(xml::openReader)) { + return SeriesSupport.copyOf(c); + } + } + + private Dataflow asDataflow(DataStructure o) { + DataflowRef ref = DataflowRef.of(o.getRef().getAgency(), o.getRef().getId(), o.getRef().getVersion()); + return Dataflow.of(ref, o.getRef(), o.getLabel()); + } +} diff --git a/sdmx-facade/sdmx-facade-util-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java new file mode 100644 index 00000000..e0d3b77d --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java @@ -0,0 +1,219 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package be.nbb.sdmx.facade.web; + +import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import be.nbb.sdmx.facade.SdmxConnection; +import be.nbb.sdmx.facade.repo.SdmxRepository; +import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxWebManagerTest { + + @Test + public void testCompliance() { + ConnectionSupplierAssert.assertCompliance(SdmxWebManager.of(repoDriver), repoSource.getName(), "123"); + } + + @Test + @SuppressWarnings("null") + public void testFactories() { + assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((Iterable) null)); + assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((SdmxWebDriver[]) null)); + + assertThat(SdmxWebManager.of()).satisfies(o -> { + assertThat(o.getDrivers()).isEmpty(); + assertThat(o.getSources()).isEmpty(); + }); + + assertThat(SdmxWebManager.of(repoDriver)).satisfies(o -> { + assertThat(o.getDrivers()).isNotEmpty(); + assertThat(o.getSources()).isNotEmpty(); + }); + } + + @Test + public void testGetDrivers() { + SdmxWebDriver driver1 = MockedWebDriver.builder().name("d1").rank(SdmxWebDriver.WRAPPED_RANK).build(); + SdmxWebDriver driver2 = MockedWebDriver.builder().name("d2").rank(SdmxWebDriver.NATIVE_RANK).build(); + + assertThat(SdmxWebManager.of(driver1, driver2).getDrivers()) + .containsExactlyElementsOf(SdmxWebManager.of(driver2, driver1).getDrivers()) + .containsExactly(driver2.getName(), driver1.getName()); + } + + @Test + public void testGetSources() { + SdmxWebSource source1 = SdmxWebSource.builder().name("source").driver("d1").endpointOf("http://abc").build(); + SdmxWebDriver driver1 = MockedWebDriver.builder().name("d1").rank(SdmxWebDriver.WRAPPED_RANK).source(source1).build(); + + SdmxWebSource source2 = SdmxWebSource.builder().name("source").driver("d2").endpointOf("http://xyz").build(); + SdmxWebDriver driver2 = MockedWebDriver.builder().name("d2").rank(SdmxWebDriver.NATIVE_RANK).source(source2).build(); + + assertThat(SdmxWebManager.of(driver1, driver2).getSources()) + .containsExactlyElementsOf(SdmxWebManager.of(driver2, driver1).getSources()) + .containsExactly(source2, source1); + } + + @Test + @SuppressWarnings("null") + public void testGetConnection() throws IOException { + SdmxWebManager manager = SdmxWebManager.of(repoDriver); + + assertThatNullPointerException().isThrownBy(() -> manager.getConnection((String) null)); + + assertThatIOException() + .isThrownBy(() -> manager.getConnection("ko")) + .as("Invalid source name"); + + assertThatCode(() -> manager.getConnection(repoSource.getName()).close()).doesNotThrowAnyException(); + + SdmxWebDriver driver1 = MockedWebDriver + .builder() + .name("d1") + .rank(SdmxWebDriver.WRAPPED_RANK) + .repo(asURL("http://abc"), repo) + .source(SdmxWebSource.builder().name("source").driver("d1").endpointOf("http://abc").build()) + .build(); + + SdmxWebDriver driver2 = MockedWebDriver + .builder() + .name("d2") + .rank(SdmxWebDriver.NATIVE_RANK) + .repo(asURL("http://xyz"), repo) + .source(SdmxWebSource.builder().name("source").driver("d2").endpointOf("http://xyz").build()) + .build(); + + try (SdmxWebConnection c = SdmxWebManager.of(driver1, driver2).getConnection("source")) { + assertThat(c.getDriver()).isEqualTo(driver2.getName()); + } + } + + @Test + @SuppressWarnings("null") + public void testGetConnectionOfSource() { + SdmxWebManager manager = SdmxWebManager.of(repoDriver); + + assertThatNullPointerException().isThrownBy(() -> manager.getConnection((SdmxWebSource) null)); + + assertThatIOException() + .isThrownBy(() -> manager.getConnection(repoSource.toBuilder().endpointOf("http://ko").build())) + .as("Invalid source endpoint"); + + assertThatIOException() + .isThrownBy(() -> manager.getConnection(repoSource.toBuilder().driver("ko").build())) + .as("Invalid source driver"); + + assertThatCode(() -> manager.getConnection(repoSource).close()).doesNotThrowAnyException(); + assertThatCode(() -> manager.getConnection(repoSource.toBuilder().name("other").build()).close()).doesNotThrowAnyException(); + } + + private final SdmxRepository repo = SdmxRepository.builder().name("repo").build(); + private final SdmxWebSource repoSource = SdmxWebSource.builder().name("repoSource").driver("repoDriver").endpoint(asURL(repo)).build(); + private final SdmxWebDriver repoDriver = MockedWebDriver + .builder() + .name("repoDriver") + .rank(0) + .repo(asURL(repo), repo) + .source(repoSource) + .build(); + + private static URL asURL(SdmxRepository o) { + try { + return new URL("http://" + o.getName()); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } + + private static URL asURL(String o) { + try { + return new URL(o); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } + + @lombok.RequiredArgsConstructor + @lombok.Builder + private static final class MockedWebDriver implements SdmxWebDriver { + + @lombok.Getter + private final String name; + + @lombok.Getter + private final int rank; + + @lombok.Singular + private final Map repos; + + @lombok.Singular + private final List sources; + + @Override + public SdmxWebConnection connect(SdmxWebSource source, SdmxWebContext context) throws IOException { + return connect(source.getEndpoint()); + } + + @Override + public Collection getDefaultSources() { + return sources; + } + + @Override + public Collection getSupportedProperties() { + return Collections.emptyList(); + } + + private SdmxWebConnection connect(URL endpoint) throws IOException { + SdmxRepository result = repos.get(endpoint); + if (result != null) { + return new MockedWebConnection(name, result.asConnection()); + } + throw new IOException(endpoint.toString()); + } + } + + @lombok.RequiredArgsConstructor + private static final class MockedWebConnection implements SdmxWebConnection { + + @lombok.Getter + private final String driver; + + @lombok.experimental.Delegate + private final SdmxConnection delegate; + + @Override + public Duration ping() throws IOException { + return Duration.ZERO; + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/util/drivers/InseeDialectTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/util/drivers/InseeDialectTest.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/internal/util/drivers/InseeDialectTest.java diff --git a/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/util/drivers/SdmxWebResourceTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/util/drivers/SdmxWebResourceTest.java new file mode 100644 index 00000000..78c478be --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/util/drivers/SdmxWebResourceTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.util.drivers; + +import be.nbb.sdmx.facade.web.SdmxWebSource; +import java.io.IOException; +import static org.assertj.core.api.Assertions.*; +import org.junit.Test; + +/** + * + * @author Philippe Charles + */ +public class SdmxWebResourceTest { + + @Test + public void testParser() throws IOException { + String xml = "\n" + + " \n" + + " ECB\n" + + " European Central Bank\n" + + " web-ri:sdmx21\n" + + " https://sdw-wsrest.ecb.europa.eu/service\n" + + " \n" + + " \n" + + ""; + + assertThat(SdmxWebResource.getParser().parseChars(xml)) + .hasSize(1) + .element(0) + .isEqualToComparingFieldByField(SdmxWebSource + .builder() + .name("ECB") + .description("European Central Bank") + .driver("web-ri:sdmx21") + .endpointOf("https://sdw-wsrest.ecb.europa.eu/service") + .property("seriesKeysOnlySupported", "true") + .build() + ); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedWebClientTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/CachedWebClientTest.java similarity index 91% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedWebClientTest.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/CachedWebClientTest.java index 0d0c9178..13126304 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/CachedWebClientTest.java +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/CachedWebClientTest.java @@ -16,8 +16,8 @@ */ package internal.web; -import _test.client.CallStackWebClient; -import be.nbb.sdmx.facade.DataQuery; +import _test.client.XCallStackWebClient; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.util.TypedId; import java.io.IOException; @@ -32,7 +32,7 @@ import _test.samples.FacadeResource; import static _test.samples.FacadeResource.ECB_FLOW_REF; import static _test.samples.FacadeResource.ECB_STRUCT_REF; -import _test.client.RepoWebClient; +import _test.client.XRepoWebClient; /** * @@ -139,28 +139,30 @@ public void testLoadData() throws IOException { CachedWebClient target = new CachedWebClient(getClient(count), "", cache, clock, 100); - assertThatNullPointerException().isThrownBy(() -> target.getData(null, query, null)); + assertThatNullPointerException().isThrownBy(() -> target.getData(null, null)); - target.getData(ECB_FLOW_REF, query, null); + DataRequest request = new DataRequest(ECB_FLOW_REF, Key.ALL, filter); + + target.getData(request, null); assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); - target.getData(ECB_FLOW_REF, query, null); + target.getData(request, null); assertThat(count).hasValue(1); assertThat(cache).containsOnlyKeys(keysId); clock.plus(100); - target.getData(ECB_FLOW_REF, query, null); + target.getData(request, null); assertThat(count).hasValue(2); assertThat(cache).containsOnlyKeys(keysId); cache.clear(); - target.getData(ECB_FLOW_REF, query, null); + target.getData(request, null); assertThat(count).hasValue(3); assertThat(cache).containsOnlyKeys(keysId); } - private final DataQuery query = DataQuery.of(Key.ALL, true); + private final DataFilter filter = DataFilter.SERIES_KEYS_ONLY; private final TypedId flowsId = TypedId.of("flows://"); private final TypedId flowId = TypedId.of("flow://").with(ECB_FLOW_REF); private final TypedId structId = TypedId.of("struct://").with(ECB_STRUCT_REF); @@ -191,7 +193,7 @@ public Instant instant() { } private static SdmxWebClient getClient(AtomicInteger count) throws IOException { - SdmxWebClient original = RepoWebClient.of(FacadeResource.ecb()); - return CallStackWebClient.of(original, count); + SdmxWebClient original = XRepoWebClient.of(FacadeResource.ecb()); + return XCallStackWebClient.of(original, count); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeWebClientTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/FailsafeWebClientTest.java similarity index 52% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeWebClientTest.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/FailsafeWebClientTest.java index cd68d546..7d1303bf 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/FailsafeWebClientTest.java +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/FailsafeWebClientTest.java @@ -16,94 +16,92 @@ */ package internal.web; -import be.nbb.sdmx.facade.DataQuery; +import _test.client.XFailingWebClient; +import _test.client.XRepoWebClient; +import _test.samples.FacadeResource; +import static _test.samples.FacadeResource.ECB_FLOW_REF; +import static _test.samples.FacadeResource.ECB_STRUCT_REF; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Key; -import be.nbb.sdmx.facade.util.UnexpectedIOException; import ioutil.IO; import java.io.IOException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; -import _test.samples.FacadeResource; -import static _test.samples.FacadeResource.ECB_FLOW_REF; -import static _test.samples.FacadeResource.ECB_STRUCT_REF; -import _test.client.FailingWebClient; -import _test.client.NoOpWebClient; -import _test.client.NullWebClient; -import _test.client.RepoWebClient; /** * * @author Philippe Charles */ +@lombok.extern.java.Log public class FailsafeWebClientTest { @Test public void testGetFlows() { - assertOperation(o -> FailsafeWebClient.of(o).getFlows()); + assertOperation(o -> FailsafeWebClient.of(o, log).getFlows()); } @Test public void testGetFlow() { - assertOperation(o -> FailsafeWebClient.of(o).getFlow(ECB_FLOW_REF)); + assertOperation(o -> FailsafeWebClient.of(o, log).getFlow(ECB_FLOW_REF)); } @Test public void testGetStructure() { - assertOperation(o -> FailsafeWebClient.of(o).getStructure(ECB_STRUCT_REF)); + assertOperation(o -> FailsafeWebClient.of(o, log).getStructure(ECB_STRUCT_REF)); } @Test public void testGetData() { - DataQuery all = DataQuery.of(Key.ALL, false); + DataRequest request = new DataRequest(ECB_FLOW_REF, Key.ALL, DataFilter.ALL); DataStructure struct = DataStructure.builder().ref(ECB_STRUCT_REF).label("hello").build(); - assertOperation(o -> FailsafeWebClient.of(o).getData(ECB_FLOW_REF, all, struct)); + assertOperation(o -> FailsafeWebClient.of(o, log).getData(request, struct)); } @Test public void testIsSeriesKeysOnlySupported() { - IO.Consumer op = o -> FailsafeWebClient.of(o).isSeriesKeysOnlySupported(); + IO.Consumer op = o -> FailsafeWebClient.of(o, log).isSeriesKeysOnlySupported(); - assertThatThrownBy(() -> op.acceptWithIO(NoOpWebClient.INSTANCE)) + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.EXPECTED)) .isInstanceOf(IOException.class); - assertThatThrownBy(() -> op.acceptWithIO(FailingWebClient.INSTANCE)) - .isInstanceOf(UnexpectedIOException.class) - .hasCauseInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.UNEXPECTED)) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf(RuntimeException.class); - assertThatCode(() -> op.acceptWithIO(RepoWebClient.of(FacadeResource.ecb()))) + assertThatCode(() -> op.acceptWithIO(XRepoWebClient.of(FacadeResource.ecb()))) .doesNotThrowAnyException(); } @Test public void testPeekStructureRef() { - IO.Consumer op = o -> FailsafeWebClient.of(o).peekStructureRef(ECB_FLOW_REF); + IO.Consumer op = o -> FailsafeWebClient.of(o, log).peekStructureRef(ECB_FLOW_REF); - assertThatThrownBy(() -> op.acceptWithIO(NoOpWebClient.INSTANCE)) + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.EXPECTED)) .isInstanceOf(IOException.class); - assertThatThrownBy(() -> op.acceptWithIO(FailingWebClient.INSTANCE)) - .isInstanceOf(UnexpectedIOException.class) - .hasCauseInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.UNEXPECTED)) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf(RuntimeException.class); - assertThatCode(() -> op.acceptWithIO(RepoWebClient.of(FacadeResource.ecb()))) + assertThatCode(() -> op.acceptWithIO(XRepoWebClient.of(FacadeResource.ecb()))) .doesNotThrowAnyException(); } private static void assertOperation(IO.Consumer op) { - assertThatThrownBy(() -> op.acceptWithIO(NoOpWebClient.INSTANCE)) + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.EXPECTED)) .isInstanceOf(IOException.class); - assertThatThrownBy(() -> op.acceptWithIO(FailingWebClient.INSTANCE)) - .isInstanceOf(UnexpectedIOException.class) - .hasCauseInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.UNEXPECTED)) + .isInstanceOf(IOException.class) + .hasCauseInstanceOf(RuntimeException.class); - assertThatThrownBy(() -> op.acceptWithIO(NullWebClient.INSTANCE)) - .isInstanceOf(UnexpectedIOException.class) + assertThatThrownBy(() -> op.acceptWithIO(XFailingWebClient.NULL)) + .isInstanceOf(IOException.class) .hasCauseInstanceOf(NullPointerException.class); - assertThatCode(() -> op.acceptWithIO(RepoWebClient.of(FacadeResource.ecb()))) + assertThatCode(() -> op.acceptWithIO(XRepoWebClient.of(FacadeResource.ecb()))) .doesNotThrowAnyException(); } } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java similarity index 91% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java rename to sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java index 00572a2d..3f5ad47e 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java +++ b/sdmx-facade/sdmx-facade-util-web/src/test/java/internal/web/SdmxWebConnectionImplTest.java @@ -21,7 +21,7 @@ import java.io.IOException; import org.junit.Test; import _test.samples.FacadeResource; -import _test.client.RepoWebClient; +import _test.client.XRepoWebClient; /** * @@ -32,6 +32,6 @@ public class SdmxWebConnectionImplTest { @Test public void testCompliance() throws IOException { SdmxRepository repo = FacadeResource.ecb(); - ConnectionAssert.assertCompliance(() -> SdmxWebConnectionImpl.of(RepoWebClient.of(repo)), FacadeResource.ECB_FLOW_REF); + ConnectionAssert.assertCompliance(() -> SdmxWebConnectionImpl.of(XRepoWebClient.of(repo), ""), FacadeResource.ECB_FLOW_REF); } } diff --git a/sdmx-facade/sdmx-facade-util-xml/pom.xml b/sdmx-facade/sdmx-facade-util-xml/pom.xml new file mode 100644 index 00000000..e8f541cf --- /dev/null +++ b/sdmx-facade/sdmx-facade-util-xml/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + + be.nbb.sdmx + sdmx-facade-parent + 2.2.3 + + + + sdmx-facade-util-xml + jar + + + + + com.google.code.findbugs + jsr305 + provided + + + org.projectlombok + lombok + provided + + + org.netbeans.api + org-openide-util-lookup + provided + + + + ${project.groupId} + sdmx-facade-api + + + ${project.groupId} + sdmx-facade-util + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + ${project.groupId} + sdmx-facade-samples + test + + + \ No newline at end of file diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/SdmxmlUri.java similarity index 59% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/SdmxmlUri.java index b4763853..92f5258d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/Sdmxml.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/SdmxmlUri.java @@ -23,14 +23,19 @@ * * @author Philippe Charles */ -@lombok.experimental.UtilityClass -public class Sdmxml { +public enum SdmxmlUri { - public static final URI NS_V10_URI = URI.create("http://www.SDMX.org/resources/SDMXML/schemas/v1_0/message"); - public static final URI NS_V20_URI = URI.create("http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message"); - public static final URI NS_V21_URI = URI.create("http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"); + NS_V10_URI("http://www.SDMX.org/resources/SDMXML/schemas/v1_0/message"), + NS_V20_URI("http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message"), + NS_V21_URI("http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"); - public boolean equals(@Nonnull URI expected, @Nonnull URI found) { - return expected.getRawSchemeSpecificPart().equalsIgnoreCase(found.getRawSchemeSpecificPart()); + private final URI uri; + + private SdmxmlUri(String uri) { + this.uri = URI.create(uri); + } + + public boolean is(@Nonnull URI found) { + return uri.getRawSchemeSpecificPart().equalsIgnoreCase(found.getRawSchemeSpecificPart()); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/AttributesBuilder.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/AttributesBuilder.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/AttributesBuilder.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/AttributesBuilder.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java similarity index 75% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java index ce57cefb..44af85dc 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/SdmxXmlStreams.java @@ -38,12 +38,7 @@ public class SdmxXmlStreams { @Nonnull - public Stax.StreamParser compactData20(@Nonnull DataStructure dsd) throws IOException { - return compactData20(dsd, DataFactory.sdmx20()); - } - - @Nonnull - public Stax.StreamParser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Xml.Parser compactData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return Stax.StreamParser.builder() .factory(StaxUtil::getInputFactoryWithoutNamespace) .handler((o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId())) @@ -51,12 +46,7 @@ public Stax.StreamParser compactData20(@Nonnull DataStructure dsd, @ } @Nonnull - public Stax.StreamParser compactData21(@Nonnull DataStructure dsd) throws IOException { - return compactData21(dsd, DataFactory.sdmx21()); - } - - @Nonnull - public Stax.StreamParser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Xml.Parser compactData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return Stax.StreamParser.builder() .factory(StaxUtil::getInputFactoryWithoutNamespace) .handler((o, onClose) -> new XMLStreamCompactDataCursor(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd), dsd.getTimeDimensionId(), dsd.getPrimaryMeasureId())) @@ -64,12 +54,7 @@ public Stax.StreamParser compactData21(@Nonnull DataStructure dsd, @ } @Nonnull - public Stax.StreamParser genericData20(@Nonnull DataStructure dsd) throws IOException { - return genericData20(dsd, DataFactory.sdmx20()); - } - - @Nonnull - public Stax.StreamParser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Xml.Parser genericData20(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return Stax.StreamParser.builder() .factory(StaxUtil::getInputFactoryWithoutNamespace) .handler((o, onClose) -> XMLStreamGenericDataCursor.sdmx20(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd))) @@ -77,12 +62,7 @@ public Stax.StreamParser genericData20(@Nonnull DataStructure dsd, @ } @Nonnull - public Stax.StreamParser genericData21(@Nonnull DataStructure dsd) throws IOException { - return genericData21(dsd, DataFactory.sdmx21()); - } - - @Nonnull - public Stax.StreamParser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { + public Xml.Parser genericData21(@Nonnull DataStructure dsd, @Nonnull DataFactory df) throws IOException { return Stax.StreamParser.builder() .factory(StaxUtil::getInputFactoryWithoutNamespace) .handler((o, onClose) -> XMLStreamGenericDataCursor.sdmx21(o, onClose, Key.builder(dsd), new ObsParser(df::getPeriodParser, df.getValueParser()), df.getFreqParser(dsd))) diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/TextBuilder.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/TextBuilder.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/TextBuilder.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/TextBuilder.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursor.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java similarity index 96% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java index fb3bd70e..9dd20161 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21.java @@ -21,13 +21,12 @@ import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.xml.Sdmxml; +import static be.nbb.sdmx.facade.xml.SdmxmlUri.NS_V21_URI; import java.util.ArrayList; import java.util.List; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.*; -import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; import java.net.URI; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; @@ -80,7 +79,7 @@ public List parse(@Nonnull XMLStreamReader reader) throws XMLStreamExc private void parseHeader(XMLStreamReader reader) throws XMLStreamException { String ns = reader.getNamespaceURI(); - check(Sdmxml.equals(NS_V21_URI, URI.create(ns)), reader, "Invalid namespace '%s'", ns); + check(NS_V21_URI.is(URI.create(ns)), reader, "Invalid namespace '%s'", ns); } private void parseStructures(XMLStreamReader reader, List flows) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursor.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java similarity index 98% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java index a5ab8894..d963dbab 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20.java @@ -21,7 +21,6 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.xml.Sdmxml; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,11 +30,11 @@ import javax.xml.stream.XMLStreamReader; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTags; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; -import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V20_URI; +import static be.nbb.sdmx.facade.xml.SdmxmlUri.NS_V20_URI; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; -import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; import java.net.URI; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * @@ -105,7 +104,7 @@ public List parse(@Nonnull XMLStreamReader reader) throws XMLStre private void parseHeader(XMLStreamReader reader) throws XMLStreamException { String ns = reader.getNamespaceURI(); - check(Sdmxml.equals(NS_V20_URI, URI.create(ns)), reader, "Invalid namespace '%s'", ns); + check(NS_V20_URI.is(URI.create(ns)), reader, "Invalid namespace '%s'", ns); } private void parseCodelists(XMLStreamReader reader, Map> codelists) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java similarity index 98% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java index 92a6100b..ac929ed5 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21.java @@ -21,7 +21,6 @@ import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dimension; import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.xml.Sdmxml; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,11 +30,11 @@ import javax.xml.stream.XMLStreamReader; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTags; import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.nextTag; -import static be.nbb.sdmx.facade.xml.Sdmxml.NS_V21_URI; +import static be.nbb.sdmx.facade.xml.SdmxmlUri.NS_V21_URI; import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; -import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; import java.net.URI; +import static be.nbb.sdmx.facade.xml.stream.XMLStreamUtil.check; /** * @@ -99,7 +98,7 @@ public List parse(@Nonnull XMLStreamReader reader) throws XMLStre private void parseHeader(XMLStreamReader reader) throws XMLStreamException { String ns = reader.getNamespaceURI(); - check(Sdmxml.equals(NS_V21_URI, URI.create(ns)), reader, "Invalid namespace '%s'", ns); + check(NS_V21_URI.is(URI.create(ns)), reader, "Invalid namespace '%s'", ns); } private void parseStructures(XMLStreamReader reader, List structs) throws XMLStreamException { diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtil.java diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/StaxUtil.java b/sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/util/StaxUtil.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/util/StaxUtil.java rename to sdmx-facade/sdmx-facade-util-xml/src/main/java/be/nbb/util/StaxUtil.java diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/TextBuilderTest.java diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java index 9ff9ac32..4ecece8a 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamCompactDataCursorTest.java @@ -33,7 +33,7 @@ /** * - * @author charphi + * @author Philippe Charles */ public class XMLStreamCompactDataCursorTest { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamFlow21Test.java diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java similarity index 97% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java index 88753dbd..9951bb1b 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java +++ b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamGenericDataCursorTest.java @@ -33,7 +33,7 @@ /** * - * @author charphi + * @author Philippe Charles */ public class XMLStreamGenericDataCursorTest { diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure20Test.java diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamStructure21Test.java diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java b/sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java rename to sdmx-facade/sdmx-facade-util-xml/src/test/java/be/nbb/sdmx/facade/xml/stream/XMLStreamUtilTest.java diff --git a/sdmx-facade/sdmx-facade-util/pom.xml b/sdmx-facade/sdmx-facade-util/pom.xml index 3fac1a97..72219643 100644 --- a/sdmx-facade/sdmx-facade-util/pom.xml +++ b/sdmx-facade/sdmx-facade-util/pom.xml @@ -5,7 +5,7 @@ be.nbb.sdmx sdmx-facade-parent - 2.2.2 + 2.2.3 sdmx-facade-util diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java index 21d56683..77f098fd 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/repo/SdmxRepository.java @@ -21,20 +21,22 @@ import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructureRef; +import be.nbb.sdmx.facade.Key; import be.nbb.sdmx.facade.SdmxConnection; import be.nbb.sdmx.facade.util.SdmxExceptions; import be.nbb.sdmx.facade.util.SeriesSupport; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -55,7 +57,7 @@ public class SdmxRepository { @lombok.NonNull @lombok.Singular - Set flows; + List flows; @lombok.NonNull Map> data; @@ -87,31 +89,38 @@ public Optional getFlow(@Nonnull DataflowRef ref) { } @Nonnull - public Optional getCursor(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { - return getData(flowRef).map(toCursor(query)); + public Optional> getData(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) { + return getDataByFlowRef(flowRef).map(toStream(key, filter)).map(o -> o.collect(Collectors.toList())); } @Nonnull - public Optional> getStream(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) { - return getData(flowRef).map(toStream(query)); + public Optional> getDataStream(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) { + return getDataByFlowRef(flowRef).map(toStream(key, filter)); } @Nonnull - private Optional> getData(@Nonnull DataflowRef flowRef) { + public Optional getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull Key key, @Nonnull DataFilter filter) { + return getDataByFlowRef(flowRef).map(toCursor(key, filter)); + } + + @Nonnull + private Optional> getDataByFlowRef(@Nonnull DataflowRef flowRef) { Objects.requireNonNull(flowRef); return Optional.ofNullable(data.get(flowRef)); } @Nonnull - private static Function, DataCursor> toCursor(@Nonnull DataQuery query) { - Objects.requireNonNull(query); - return o -> SeriesSupport.asCursor(o, query.getKey()); + private static Function, DataCursor> toCursor(@Nonnull Key key, @Nonnull DataFilter filter) { + Objects.requireNonNull(key); + Objects.requireNonNull(filter); + return o -> SeriesSupport.asCursor(o, key); } @Nonnull - private static Function, Stream> toStream(@Nonnull DataQuery query) { - Objects.requireNonNull(query); - return o -> o.stream().filter(s -> query.getKey().contains(s.getKey())); + private static Function, Stream> toStream(@Nonnull Key key, @Nonnull DataFilter filter) { + Objects.requireNonNull(key); + Objects.requireNonNull(filter); + return o -> o.stream().filter(s -> key.contains(s.getKey())); } public static final class Builder { @@ -154,7 +163,7 @@ private RepoConnection(SdmxRepository repo) { } @Override - public Set getFlows() throws IOException { + public Collection getFlows() throws IOException { checkState(); return repo.getFlows(); } @@ -164,7 +173,7 @@ public Dataflow getFlow(DataflowRef flowRef) throws IOException { checkState(); return repo .getFlow(flowRef) - .orElseThrow(() -> SdmxExceptions.missingFlow(flowRef)); + .orElseThrow(() -> SdmxExceptions.missingFlow(repo.getName(), flowRef)); } @Override @@ -173,23 +182,31 @@ public DataStructure getStructure(DataflowRef flowRef) throws IOException { DataStructureRef structRef = getFlow(flowRef).getStructureRef(); return repo .getStructure(structRef) - .orElseThrow(() -> SdmxExceptions.missingStructure(structRef)); + .orElseThrow(() -> SdmxExceptions.missingStructure(repo.getName(), structRef)); + } + + @Override + public List getData(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { + checkState(); + return repo + .getData(flowRef, key, filter) + .orElseThrow(() -> SdmxExceptions.missingData(repo.getName(), flowRef)); } @Override - public DataCursor getCursor(DataflowRef flowRef, DataQuery query) throws IOException { + public Stream getDataStream(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { checkState(); return repo - .getCursor(flowRef, query) - .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); + .getDataStream(flowRef, key, filter) + .orElseThrow(() -> SdmxExceptions.missingData(repo.getName(), flowRef)); } @Override - public Stream getStream(DataflowRef flowRef, DataQuery query) throws IOException { + public DataCursor getDataCursor(DataflowRef flowRef, Key key, DataFilter filter) throws IOException { checkState(); return repo - .getStream(flowRef, query) - .orElseThrow(() -> SdmxExceptions.missingData(flowRef)); + .getDataCursor(flowRef, key, filter) + .orElseThrow(() -> SdmxExceptions.missingData(repo.getName(), flowRef)); } @Override @@ -204,7 +221,7 @@ public void close() throws IOException { private void checkState() throws IOException { if (closed) { - throw SdmxExceptions.connectionClosed(); + throw SdmxExceptions.connectionClosed(repo.getName()); } } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java index da986017..8fb7d1c5 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxExceptions.java @@ -29,22 +29,22 @@ public final class SdmxExceptions { @Nonnull - public IOException connectionClosed() { - return new IOException("Connection already closed"); + public IOException connectionClosed(@Nonnull String source) { + return new IOException("Connection '" + source + "' already closed"); } @Nonnull - public IOException missingFlow(@Nonnull DataflowRef ref) { - return new IOException("Missing dataflow '" + ref + "'"); + public IOException missingFlow(@Nonnull String source, @Nonnull DataflowRef ref) { + return new IOException("Missing dataflow '" + ref + "' in '" + source + "'"); } @Nonnull - public IOException missingStructure(@Nonnull DataStructureRef ref) { - return new IOException("Missing datastructure '" + ref + "'"); + public IOException missingStructure(@Nonnull String source, @Nonnull DataStructureRef ref) { + return new IOException("Missing datastructure '" + ref + "' in '" + source + "'"); } @Nonnull - public IOException missingData(@Nonnull DataflowRef ref) { - return new IOException("Missing data '" + ref + "'"); + public IOException missingData(@Nonnull String source, @Nonnull DataflowRef ref) { + return new IOException("Missing data '" + ref + "' in '" + source + "'"); } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java index a4bce4be..34a82b4d 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SdmxFix.java @@ -28,7 +28,13 @@ @Documented public @interface SdmxFix { - String id(); + int id(); + + Category category() default Category.NONE; String cause(); + + enum Category { + NONE, ENDPOINT, QUERY, MEDIA_TYPE, CONTENT; + } } diff --git a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java index 312df544..b896ac21 100644 --- a/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java +++ b/sdmx-facade/sdmx-facade-util/src/main/java/be/nbb/sdmx/facade/util/SeriesSupport.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -72,10 +73,10 @@ public static List copyOfKeysAndMeta(@Nonnull DataCursor cursor) throws } @Nonnull - public DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { - Objects.requireNonNull(list); - Objects.requireNonNull(ref); - return new SeriesCursor(list, ref); + public List asList(@Nonnull IO.Supplier source) throws IOException { + try (Stream stream = IO.Stream.open(source, SeriesSupport::getDataStream)) { + return stream.collect(Collectors.toList()); + } } @Nonnull @@ -83,6 +84,13 @@ public Stream asStream(@Nonnull IO.Supplier source) throws I return IO.Stream.open(source, SeriesSupport::getDataStream); } + @Nonnull + public DataCursor asCursor(@Nonnull List list, @Nonnull Key ref) { + Objects.requireNonNull(list); + Objects.requireNonNull(ref); + return new SeriesCursor(list, ref); + } + @SuppressWarnings("null") private Stream getDataStream(DataCursor cursor) { Series.Builder builder = Series.builder(); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java index 096f2a3a..f6dd75b8 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/repo/SdmxRepositoryTest.java @@ -16,7 +16,7 @@ */ package be.nbb.sdmx.facade.repo; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; @@ -57,13 +57,14 @@ public void testBuilder() { @Test @SuppressWarnings("null") public void testGetCursor() { - assertThatNullPointerException().isThrownBy(() -> repo.getCursor(null, all)); - assertThatNullPointerException().isThrownBy(() -> repo.getCursor(goodFlowRef, null)); + assertThatNullPointerException().isThrownBy(() -> repo.getDataCursor(null, Key.ALL, DataFilter.ALL)); + assertThatNullPointerException().isThrownBy(() -> repo.getDataCursor(goodFlowRef, null, DataFilter.ALL)); + assertThatNullPointerException().isThrownBy(() -> repo.getDataCursor(goodFlowRef, Key.ALL, null)); - DataCursorAssert.assertCompliance(() -> repo.getCursor(goodFlowRef, all).get()); + DataCursorAssert.assertCompliance(() -> repo.getDataCursor(goodFlowRef, Key.ALL, DataFilter.ALL).get()); - assertThat(repo.getCursor(goodFlowRef, all)).isNotEmpty(); - assertThat(repo.getCursor(badFlowRef, all)).isEmpty(); + assertThat(repo.getDataCursor(goodFlowRef, Key.ALL, DataFilter.ALL)).isNotEmpty(); + assertThat(repo.getDataCursor(badFlowRef, Key.ALL, DataFilter.ALL)).isEmpty(); } @Test @@ -93,10 +94,11 @@ public void testGetFlows() { @Test @SuppressWarnings("null") public void testGetStream() { - assertThatNullPointerException().isThrownBy(() -> repo.getStream(null, all)); - assertThatNullPointerException().isThrownBy(() -> repo.getStream(goodFlowRef, null)); - assertThat(repo.getStream(goodFlowRef, all)).isNotEmpty(); - assertThat(repo.getStream(badFlowRef, all)).isEmpty(); + assertThatNullPointerException().isThrownBy(() -> repo.getDataStream(null, Key.ALL, DataFilter.ALL)); + assertThatNullPointerException().isThrownBy(() -> repo.getDataStream(goodFlowRef, null, DataFilter.ALL)); + assertThatNullPointerException().isThrownBy(() -> repo.getDataStream(goodFlowRef, Key.ALL, null)); + assertThat(repo.getDataStream(goodFlowRef, Key.ALL, DataFilter.ALL)).isNotEmpty(); + assertThat(repo.getDataStream(badFlowRef, Key.ALL, DataFilter.ALL)).isEmpty(); } @Test @@ -109,7 +111,6 @@ public void testGetStructure() { private final DataStructureRef goodStructRef = DataStructureRef.of("NBB","goodStruct", "v1.0"); private final DataStructureRef badStructRef = DataStructureRef.parse("badStruct"); - private final DataQuery all = DataQuery.of(Key.ALL, false); private final DataflowRef goodFlowRef = DataflowRef.of("NBB", "XYZ", "v2.0"); private final DataflowRef badFlowRef = DataflowRef.parse("other"); private final Dataflow flow = Dataflow.of(goodFlowRef, goodStructRef, "flow1"); diff --git a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java index 07507159..c4719ca9 100644 --- a/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java +++ b/sdmx-facade/sdmx-facade-util/src/test/java/be/nbb/sdmx/facade/util/SeriesSupportTest.java @@ -17,19 +17,15 @@ package be.nbb.sdmx.facade.util; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.Frequency; import be.nbb.sdmx.facade.Key; -import static be.nbb.sdmx.facade.LanguagePriorityList.ANY; import be.nbb.sdmx.facade.Obs; import be.nbb.sdmx.facade.Series; -import be.nbb.sdmx.facade.samples.SdmxSource; import be.nbb.sdmx.facade.tck.DataCursorAssert; -import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import java.io.IOException; import java.time.LocalDateTime; +import java.util.Arrays; import java.util.Collections; -import java.util.List; import javax.xml.stream.XMLStreamException; import static org.assertj.core.api.Assertions.*; import org.junit.Test; @@ -70,13 +66,12 @@ public void testCopyOf() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOf(c)).isEmpty(); } - try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(series), Key.ALL)) { - assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(series); + try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(s1), Key.ALL)) { + assertThat(SeriesSupport.copyOf(c)).hasSize(1).element(0).isSameAs(s1); } - List dsds = SdmxXmlStreams.struct21(ANY).parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.ECB_DATA::openReader)) { - assertThat(SeriesSupport.copyOf(c)).hasSize(120); + try (DataCursor c = SeriesSupport.asCursor(Arrays.asList(s1, s2), Key.ALL)) { + assertThat(SeriesSupport.copyOf(c)).hasSize(2); } } @@ -89,21 +84,21 @@ public void testCopyOfKeysAndMeta() throws IOException, XMLStreamException { assertThat(SeriesSupport.copyOfKeysAndMeta(c)).isEmpty(); } - try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(series), Key.ALL)) { + try (DataCursor c = SeriesSupport.asCursor(Collections.singletonList(s1), Key.ALL)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) .hasSize(1) .element(0) - .isEqualToComparingOnlyGivenFields(series, "key", "freq", "meta"); + .isEqualToComparingOnlyGivenFields(s1, "key", "freq", "meta"); } - List dsds = SdmxXmlStreams.struct21(ANY).parseReader(SdmxSource.ECB_DATA_STRUCTURE::openReader); - try (DataCursor c = SdmxXmlStreams.genericData21(dsds.get(0)).parseReader(SdmxSource.ECB_DATA::openReader)) { + try (DataCursor c = SeriesSupport.asCursor(Arrays.asList(s1, s2), Key.ALL)) { assertThat(SeriesSupport.copyOfKeysAndMeta(c)) .allMatch(o -> o.getObs().isEmpty()) - .hasSize(120); + .hasSize(2); } } - private final Series series = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); + private final Series s1 = Series.builder().key(Key.of("BE")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); + private final Series s2 = Series.builder().key(Key.of("FR")).freq(Frequency.MONTHLY).obs(Obs.of(LocalDateTime.now(), Math.PI)).meta("hello", "world").build(); } diff --git a/sdmx-facade/sdmx-facade-web-ri/pom.xml b/sdmx-facade/sdmx-facade-web-ri/pom.xml new file mode 100644 index 00000000..0d60a6d8 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web-ri/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + be.nbb.sdmx + sdmx-facade-parent + 2.2.3 + + + sdmx-facade-web-ri + jar + + + + + com.google.code.findbugs + jsr305 + provided + + + org.projectlombok + lombok + provided + + + org.netbeans.api + org-openide-util-lookup + provided + + + + ${project.groupId} + sdmx-facade-api + + + ${project.groupId} + sdmx-facade-util + + + ${project.groupId} + sdmx-facade-util-web + + + ${project.groupId} + sdmx-facade-util-xml + + + + junit + junit + test + + + org.assertj + assertj-core + test + + + ${project.groupId} + sdmx-facade-samples + test + + + \ No newline at end of file diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestClient.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestClient.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestClient.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestClient.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestClientImpl.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestClientImpl.java similarity index 96% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestClientImpl.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestClientImpl.java index 5e11b859..b31dc4c7 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestClientImpl.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestClientImpl.java @@ -62,6 +62,9 @@ public final class RestClientImpl implements RestClient { @lombok.NonNull private final EventListener listener; + @lombok.NonNull + private final IO.Consumer validator; + @Override public InputStream openStream(URL query, String mediaType, String langs) throws IOException { listener.onOpenStream(query, mediaType, langs); @@ -95,6 +98,7 @@ private InputStream openStream(URL query, String mediaType, String langs, int re case 308: // Permanent Redirect return redirect(http, mediaType, langs, redirects); case HttpURLConnection.HTTP_OK: + validator.acceptWithIO(http); return getBody(http); default: throw getError(http); @@ -104,7 +108,9 @@ private InputStream openStream(URL query, String mediaType, String langs, int re private Proxy getProxy(URL url) throws IOException { try { List proxies = proxySelector.select(url.toURI()); - return proxies.isEmpty() ? Proxy.NO_PROXY : proxies.get(0); + Proxy result = proxies.isEmpty() ? Proxy.NO_PROXY : proxies.get(0); + listener.onProxy(url, result); + return result; } catch (URISyntaxException ex) { throw new IOException(ex); } @@ -156,6 +162,8 @@ public interface EventListener { void onOpenStream(URL query, String mediaType, String langs); void onRedirection(URL oldUrl, URL newUrl); + + void onProxy(URL query, Proxy proxy); } @lombok.Getter diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestQueryBuilder.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestQueryBuilder.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/util/rest/RestQueryBuilder.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/util/rest/RestQueryBuilder.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbsDriver2.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbsDriver2.java similarity index 67% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbsDriver2.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbsDriver2.java index 420e055d..ff6eebbf 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbsDriver2.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbsDriver2.java @@ -16,40 +16,43 @@ */ package internal.web.drivers; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.util.SdmxFix; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.QUERY; import be.nbb.sdmx.facade.web.SdmxWebSource; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.web.SdmxWebDriverSupport; import java.io.IOException; import java.net.URL; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import internal.web.DataRequest; +import internal.web.SdmxWebClient; +import org.openide.util.lookup.ServiceProvider; /** * * @author Philippe Charles */ +@ServiceProvider(service = SdmxWebDriver.class) public final class AbsDriver2 implements SdmxWebDriver { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("abs@facade") + .name("web-ri:abs") + .rank(NATIVE_RANK) .client(AbsClient2::new) - .supportedProperties(Util.CONNECTION_PROPERTIES) + .supportedProperties(RestClients.CONNECTION_PROPERTIES) .sourceOf("ABS", "Australian Bureau of Statistics", "http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") .build(); - private static final class AbsClient2 extends AbstractDotStat { + private static final class AbsClient2 extends DotStatRestClient { - private AbsClient2(SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) { - super(s.getEndpoint(), l, Util.getRestClient(s, c)); + private AbsClient2(SdmxWebSource s, SdmxWebContext c) { + super(SdmxWebClient.getClientName(s), s.getEndpoint(), c.getLanguages(), RestClients.getRestClient(s, c)); } - @SdmxFix(id = "ABS#1", cause = "Agency is required in query") + @SdmxFix(id = 1, category = QUERY, cause = "Agency is required in query") private static final String AGENCY = "ABS"; @Override @@ -58,8 +61,8 @@ protected URL getStructureQuery(DataStructureRef ref) throws IOException { } @Override - protected URL getDataQuery(DataflowRef flowRef, DataQuery query) throws IOException { - return getDataQuery(endpoint, flowRef, query).path(AGENCY).build(); + protected URL getDataQuery(DataRequest request) throws IOException { + return getDataQuery(endpoint, request).path(AGENCY).build(); } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractRestClient.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbstractRestClient.java similarity index 89% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractRestClient.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbstractRestClient.java index ce4046cd..a68931b1 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractRestClient.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/AbstractRestClient.java @@ -17,11 +17,11 @@ package internal.web.drivers; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; +import internal.web.DataRequest; import internal.web.SdmxWebClient; import java.io.IOException; import java.net.URL; @@ -35,7 +35,6 @@ * * @author Philippe Charles */ -@lombok.AllArgsConstructor abstract class AbstractRestClient implements SdmxWebClient { @Override @@ -57,8 +56,8 @@ public DataStructure getStructure(DataStructureRef ref) throws IOException { } @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - URL url = getDataQuery(flowRef, query); + public DataCursor getData(DataRequest request, DataStructure dsd) throws IOException { + URL url = getDataQuery(request); return getData(dsd, url); } @@ -89,7 +88,7 @@ public Duration ping() throws IOException { abstract protected DataStructure getStructure(@Nonnull URL url, @Nonnull DataStructureRef ref) throws IOException; @Nonnull - abstract protected URL getDataQuery(@Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException; + abstract protected URL getDataQuery(@Nonnull DataRequest request) throws IOException; @Nonnull abstract protected DataCursor getData(@Nonnull DataStructure dsd, @Nonnull URL url) throws IOException; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/DotStatDriver2.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatDriver2.java similarity index 67% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/DotStatDriver2.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatDriver2.java index a3f42b54..e95b0827 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/DotStatDriver2.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatDriver2.java @@ -16,35 +16,38 @@ */ package internal.web.drivers; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.util.SdmxFix; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.ENDPOINT; import be.nbb.sdmx.facade.web.SdmxWebSource; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.web.SdmxWebDriverSupport; import internal.web.SdmxWebClient; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import org.openide.util.lookup.ServiceProvider; /** * * @author Philippe Charles */ +@ServiceProvider(service = SdmxWebDriver.class) public final class DotStatDriver2 implements SdmxWebDriver { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("dotstat@facade") + .name("web-ri:dotstat") + .rank(NATIVE_RANK) .client(DotStatDriver2::of) - .supportedProperties(Util.CONNECTION_PROPERTIES) + .supportedProperties(RestClients.CONNECTION_PROPERTIES) .sourceOf("OECD", "The Organisation for Economic Co-operation and Development", "https://stats.oecd.org/restsdmx/sdmx.ashx") .sourceOf("SE", "Statistics Estonia", "http://andmebaas.stat.ee/restsdmx/sdmx.ashx") - .sourceOf("UIS", "Unesco Institute for Statistics", UIS_URL) + .sourceOf("UIS", "Unesco Institute for Statistics", UIS_ENDPOINT) .build(); - private static SdmxWebClient of(SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) { - return new AbstractDotStat(s.getEndpoint(), l, Util.getRestClient(s, c)); + private static SdmxWebClient of(SdmxWebSource s, SdmxWebContext c) { + return new DotStatRestClient(SdmxWebClient.getClientName(s), s.getEndpoint(), c.getLanguages(), RestClients.getRestClient(s, c)); } - @SdmxFix(id = "#UIS1", cause = "API requires auth by key in header and this is not supported yet in facade") - private final static String UIS_URL = "http://data.uis.unesco.org/RestSDMX/sdmx.ashx"; + @SdmxFix(id = 1, category = ENDPOINT, cause = "UIS API requires auth by key in header and this is not supported yet in facade") + private final static String UIS_ENDPOINT = "http://data.uis.unesco.org/RestSDMX/sdmx.ashx"; } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractDotStat.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatRestClient.java similarity index 66% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractDotStat.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatRestClient.java index b218386a..fda841f2 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractDotStat.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/DotStatRestClient.java @@ -17,18 +17,22 @@ package internal.web.drivers; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.LanguagePriorityList; +import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SdmxExceptions; +import be.nbb.sdmx.facade.util.SdmxFix; import static be.nbb.sdmx.facade.util.SdmxMediaType.XML; import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import internal.util.rest.RestClient; import internal.util.rest.RestQueryBuilder; +import internal.web.DataRequest; +import ioutil.IO; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.List; import java.util.stream.Collectors; @@ -40,12 +44,18 @@ * @author Philippe Charles */ @AllArgsConstructor -class AbstractDotStat extends AbstractRestClient { +class DotStatRestClient extends AbstractRestClient { + protected final String name; protected final URL endpoint; protected final LanguagePriorityList langs; protected final RestClient executor; + @Override + public String getName() throws IOException { + return name; + } + @Override protected URL getFlowsQuery() throws IOException { return getFlowsQuery(endpoint).build(); @@ -55,9 +65,9 @@ protected URL getFlowsQuery() throws IOException { protected List getFlows(URL url) throws IOException { return SdmxXmlStreams .struct20(langs) - .parseStream(() -> executor.openStream(url, XML, langs.toString())) + .parseStream(calling(url, XML)) .stream() - .map(AbstractDotStat::asDataflow) + .map(DotStatRestClient::getFlowFromStructure) .collect(Collectors.toList()); } @@ -70,11 +80,11 @@ protected URL getFlowQuery(DataflowRef ref) throws IOException { protected Dataflow getFlow(URL url, DataflowRef ref) throws IOException { return SdmxXmlStreams .struct20(langs) - .parseStream(() -> executor.openStream(url, XML, langs.toString())) + .parseStream(calling(url, XML)) .stream() - .map(AbstractDotStat::asDataflow) + .map(DotStatRestClient::getFlowFromStructure) .findFirst() - .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); + .orElseThrow(() -> SdmxExceptions.missingFlow(name, ref)); } @Override @@ -86,22 +96,24 @@ protected URL getStructureQuery(DataStructureRef ref) throws IOException { protected DataStructure getStructure(URL url, DataStructureRef ref) throws IOException { return SdmxXmlStreams .struct20(langs) - .parseStream(() -> executor.openStream(url, XML, langs.toString())) + .parseStream(calling(url, XML)) .stream() .findFirst() - .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); + .orElseThrow(() -> SdmxExceptions.missingStructure(name, ref)); } @Override - protected URL getDataQuery(DataflowRef flowRef, DataQuery query) throws IOException { - return getDataQuery(endpoint, flowRef, query).build(); + protected URL getDataQuery(DataRequest request) throws IOException { + return getDataQuery(endpoint, request).build(); } + @SdmxFix(id = 1, category = SdmxFix.Category.CONTENT, cause = "Time dimension is always TIME in data") @Override protected DataCursor getData(DataStructure dsd, URL url) throws IOException { + DataStructure modifiedDsd = dsd.toBuilder().timeDimensionId("TIME").build(); return SdmxXmlStreams - .compactData20(dsd) - .parseStream(() -> executor.openStream(url, XML, langs.toString())); + .compactData20(modifiedDsd, DataFactory.sdmx20()) + .parseStream(calling(url, XML)); } @Override @@ -111,51 +123,59 @@ public boolean isSeriesKeysOnlySupported() throws IOException { @Override public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { - return asDataStructureRef(flowRef); + return getStructureRefFromFlowRef(flowRef); + } + + private IO.Supplier calling(URL query, String mediaType) throws IOException { + return () -> executor.openStream(query, mediaType, langs.toString()); } @Nonnull static RestQueryBuilder getFlowsQuery(@Nonnull URL endpoint) throws IOException { - return RestQueryBuilder.of(endpoint) + return RestQueryBuilder + .of(endpoint) .path(DATASTRUCTURE_RESOURCE) .path("ALL"); } @Nonnull static RestQueryBuilder getFlowQuery(@Nonnull URL endpoint, @Nonnull DataflowRef ref) throws IOException { - return RestQueryBuilder.of(endpoint) + return RestQueryBuilder + .of(endpoint) .path(DATASTRUCTURE_RESOURCE) .path(ref.getId()); } @Nonnull static RestQueryBuilder getStructureQuery(@Nonnull URL endpoint, @Nonnull DataStructureRef ref) throws IOException { - return RestQueryBuilder.of(endpoint) + return RestQueryBuilder + .of(endpoint) .path(DATASTRUCTURE_RESOURCE) .path(ref.getId()); } @Nonnull - static RestQueryBuilder getDataQuery(@Nonnull URL endpoint, @Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException { - return RestQueryBuilder.of(endpoint) + static RestQueryBuilder getDataQuery(@Nonnull URL endpoint, @Nonnull DataRequest request) throws IOException { + return RestQueryBuilder + .of(endpoint) .path(DATA_RESOURCE) - .path(flowRef.getId()) - .path(query.getKey().toString()) + .path(request.getFlowRef().getId()) + .path(request.getKey().toString()) .param("format", "compact_v2"); } @Nonnull - static Dataflow asDataflow(@Nonnull DataStructure o) { - return Dataflow.of(asDataflowRef(o.getRef()), o.getRef(), o.getLabel()); + static Dataflow getFlowFromStructure(@Nonnull DataStructure o) { + return Dataflow.of(getFlowRefFromStructureRef(o.getRef()), o.getRef(), o.getLabel()); } @Nonnull - static DataflowRef asDataflowRef(@Nonnull DataStructureRef o) { + static DataflowRef getFlowRefFromStructureRef(@Nonnull DataStructureRef o) { return DataflowRef.of(o.getAgency(), o.getId(), o.getVersion()); } @Nonnull - static DataStructureRef asDataStructureRef(@Nonnull DataflowRef o) { + static DataStructureRef getStructureRefFromFlowRef(@Nonnull DataflowRef o) { return DataStructureRef.of(o.getAgency(), o.getId(), o.getVersion()); } diff --git a/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/NbbDriver2.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/NbbDriver2.java new file mode 100644 index 00000000..e239c003 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/NbbDriver2.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package internal.web.drivers; + +import be.nbb.sdmx.facade.util.SdmxFix; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.CONTENT; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.QUERY; +import be.nbb.sdmx.facade.web.SdmxWebSource; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import internal.util.rest.RestQueryBuilder; +import internal.web.SdmxWebDriverSupport; +import java.io.IOException; +import java.net.URL; +import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import internal.web.DataRequest; +import internal.web.SdmxWebClient; +import java.net.HttpURLConnection; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author Philippe Charles + */ +@ServiceProvider(service = SdmxWebDriver.class) +public final class NbbDriver2 implements SdmxWebDriver { + + @lombok.experimental.Delegate + private final SdmxWebDriverSupport support = SdmxWebDriverSupport + .builder() + .name("web-ri:nbb") + .rank(NATIVE_RANK) + .client(NbbClient2::new) + .supportedProperties(RestClients.CONNECTION_PROPERTIES) + .sourceOf("NBB", "National Bank of Belgium", "https://stat.nbb.be/restsdmx/sdmx.ashx") + .build(); + + private static final class NbbClient2 extends DotStatRestClient { + + private NbbClient2(SdmxWebSource s, SdmxWebContext c) { + super(SdmxWebClient.getClientName(s), s.getEndpoint(), c.getLanguages(), + RestClients.getRestClient(s, c, NbbClient2::checkResponseForError)); + } + + @SdmxFix(id = 1, category = QUERY, cause = "'/all' must be encoded to '%2Fall'") + @Override + protected URL getDataQuery(DataRequest request) throws IOException { + return RestQueryBuilder + .of(endpoint) + .path(DATA_RESOURCE) + .path(request.getFlowRef().getId()) + .path(request.getKey().toString() + "/all") + .param("format", "compact_v2") + .build(); + } + + @SdmxFix(id = 2, category = CONTENT, cause = "Some interal errors redirect to an html page") + private static void checkResponseForError(HttpURLConnection http) throws IOException { + if (http.getContentType().equals("text/html")) { + throw new IOException("Service not available"); + } + } + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Util.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/RestClients.java similarity index 75% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Util.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/RestClients.java index c8f58b68..46b9a56d 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Util.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/RestClients.java @@ -27,22 +27,30 @@ import java.util.logging.Level; import java.util.logging.Logger; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import ioutil.IO; +import java.net.HttpURLConnection; +import java.net.Proxy; /** * * @author Philippe Charles */ @lombok.experimental.UtilityClass -class Util { +class RestClients { public RestClient getRestClient(SdmxWebSource o, SdmxWebContext context) { + return getRestClient(o, context, IO.Consumer.noOp()); + } + + public RestClient getRestClient(SdmxWebSource o, SdmxWebContext context, IO.Consumer validator) { return RestClientImpl.of( SdmxWebProperty.getReadTimeout(o.getProperties()), SdmxWebProperty.getConnectTimeout(o.getProperties()), SdmxWebProperty.getMaxRedirects(o.getProperties()), context.getProxySelector(), context.getSslSocketFactory(), - new EventListenerImpl(context.getLogger()) + new Listener(context.getLogger()), + validator ); } @@ -54,19 +62,24 @@ public RestClient getRestClient(SdmxWebSource o, SdmxWebContext context) { )); @lombok.AllArgsConstructor - private static final class EventListenerImpl implements RestClientImpl.EventListener { + private static final class Listener implements RestClientImpl.EventListener { @lombok.NonNull private final Logger logger; @Override public void onOpenStream(URL query, String mediaType, String langs) { - logger.log(Level.FINE, "Querying ''{0}''", query); + logger.log(Level.INFO, "Querying ''{0}''", query); } @Override public void onRedirection(URL oldUrl, URL newUrl) { - logger.log(Level.FINE, "Redirecting to ''{0}''", newUrl); + logger.log(Level.INFO, "Redirecting to ''{0}''", newUrl); + } + + @Override + public void onProxy(URL query, Proxy proxy) { + logger.log(Level.INFO, "Using proxy ''{0}''", proxy); } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Sdmx21Driver2.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21Driver2.java similarity index 76% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Sdmx21Driver2.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21Driver2.java index beca817b..9d3d7111 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/Sdmx21Driver2.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21Driver2.java @@ -16,7 +16,6 @@ */ package internal.web.drivers; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.parser.DataFactory; import static internal.web.SdmxWebProperty.*; import be.nbb.sdmx.facade.web.SdmxWebSource; @@ -26,25 +25,32 @@ import internal.web.SdmxWebClient; import internal.web.SdmxWebProperty; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import org.openide.util.lookup.ServiceProvider; /** * * @author Philippe Charles */ +@ServiceProvider(service = SdmxWebDriver.class) public final class Sdmx21Driver2 implements SdmxWebDriver { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("sdmx21@facade") + .name("web-ri:sdmx21") + .rank(NATIVE_RANK) .client(Sdmx21Driver2::of) - .supportedProperties(Util.CONNECTION_PROPERTIES) + .supportedProperties(RestClients.CONNECTION_PROPERTIES) .supportedProperty(SERIES_KEYS_ONLY_SUPPORTED_PROPERTY) .sources(SdmxWebResource.load("/internal/web/drivers/sdmx21.xml")) .build(); - private static SdmxWebClient of(SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) { - return new AbstractSdmx21(s.getEndpoint(), l, Util.getRestClient(s, c), + private static SdmxWebClient of(SdmxWebSource s, SdmxWebContext c) { + return new Sdmx21RestClient( + SdmxWebClient.getClientName(s), + s.getEndpoint(), + c.getLanguages(), + RestClients.getRestClient(s, c), SdmxWebProperty.isSeriesKeysOnlySupported(s.getProperties()), DataFactory.sdmx21() ); diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractSdmx21.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21RestClient.java similarity index 83% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractSdmx21.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21RestClient.java index 234ad39f..fb0fd673 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/AbstractSdmx21.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/Sdmx21RestClient.java @@ -17,7 +17,6 @@ package internal.web.drivers; import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructure; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.Dataflow; @@ -28,7 +27,7 @@ import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SdmxExceptions; import static be.nbb.sdmx.facade.util.SdmxMediaType.*; -import static be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams.*; +import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams; import ioutil.IO; import java.io.IOException; import java.io.InputStream; @@ -36,6 +35,7 @@ import java.util.List; import internal.util.rest.RestClient; import internal.util.rest.RestQueryBuilder; +import internal.web.DataRequest; import javax.annotation.Nonnull; /** @@ -43,14 +43,20 @@ * @author Philippe Charles */ @lombok.RequiredArgsConstructor -class AbstractSdmx21 extends AbstractRestClient { +class Sdmx21RestClient extends AbstractRestClient { + protected final String name; protected final URL endpoint; protected final LanguagePriorityList langs; protected final RestClient executor; protected final boolean seriesKeysOnlySupported; protected final DataFactory dialect; + @Override + public String getName() throws IOException { + return name; + } + @Override protected URL getFlowsQuery() throws IOException { return getFlowsQuery(endpoint).build(); @@ -58,7 +64,8 @@ protected URL getFlowsQuery() throws IOException { @Override protected List getFlows(URL url) throws IOException { - return flow21(langs) + return SdmxXmlStreams + .flow21(langs) .parseStream(calling(url, XML)); } @@ -69,12 +76,13 @@ protected URL getFlowQuery(DataflowRef ref) throws IOException { @Override protected Dataflow getFlow(URL url, DataflowRef ref) throws IOException { - return flow21(langs) + return SdmxXmlStreams + .flow21(langs) .parseStream(calling(url, XML)) .stream() .filter(ref::containsRef) .findFirst() - .orElseThrow(() -> SdmxExceptions.missingFlow(ref)); + .orElseThrow(() -> SdmxExceptions.missingFlow(name, ref)); } @Override @@ -84,22 +92,24 @@ protected URL getStructureQuery(DataStructureRef ref) throws IOException { @Override protected DataStructure getStructure(URL url, DataStructureRef ref) throws IOException { - return struct21(langs) + return SdmxXmlStreams + .struct21(langs) .parseStream(calling(url, STRUCTURE_21)) .stream() .filter(ref::equalsRef) .findFirst() - .orElseThrow(() -> SdmxExceptions.missingStructure(ref)); + .orElseThrow(() -> SdmxExceptions.missingStructure(name, ref)); } @Override - protected URL getDataQuery(DataflowRef flowRef, DataQuery query) throws IOException { - return getDataQuery(endpoint, flowRef, query).build(); + protected URL getDataQuery(DataRequest request) throws IOException { + return getDataQuery(endpoint, request).build(); } @Override protected DataCursor getData(DataStructure dsd, URL url) throws IOException { - return compactData21(dsd, dialect) + return SdmxXmlStreams + .compactData21(dsd, dialect) .parseStream(calling(url, STRUCTURE_SPECIFIC_DATA_21)); } @@ -119,7 +129,8 @@ private IO.Supplier calling(URL query, String mediaType) @Nonnull static RestQueryBuilder onMeta(@Nonnull URL endpoint, @Nonnull String resourceType, @Nonnull ResourceRef ref) { - return RestQueryBuilder.of(endpoint) + return RestQueryBuilder + .of(endpoint) .path(resourceType) .path(ref.getAgency()) .path(ref.getId()) @@ -128,7 +139,8 @@ static RestQueryBuilder onMeta(@Nonnull URL endpoint, @Nonnull String resourceTy @Nonnull static RestQueryBuilder onData(@Nonnull URL endpoint, @Nonnull DataflowRef flowRef, @Nonnull Key key) { - return RestQueryBuilder.of(endpoint) + return RestQueryBuilder + .of(endpoint) .path(DATA_RESOURCE) .path(flowRef.toString()) .path(key.toString()) @@ -151,9 +163,9 @@ static RestQueryBuilder getStructureQuery(@Nonnull URL endpoint, @Nonnull DataSt } @Nonnull - static RestQueryBuilder getDataQuery(@Nonnull URL endpoint, @Nonnull DataflowRef flowRef, @Nonnull DataQuery query) throws IOException { - RestQueryBuilder result = onData(endpoint, flowRef, query.getKey()); - switch (query.getDetail()) { + static RestQueryBuilder getDataQuery(@Nonnull URL endpoint, @Nonnull DataRequest request) throws IOException { + RestQueryBuilder result = onData(endpoint, request.getFlowRef(), request.getKey()); + switch (request.getFilter().getDetail()) { case SERIES_KEYS_ONLY: result.param(DETAIL_PARAM, "serieskeysonly"); break; diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/WbDriver2.java b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/WbDriver2.java similarity index 70% rename from sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/WbDriver2.java rename to sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/WbDriver2.java index 06b9b148..be1687cc 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/WbDriver2.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/java/internal/web/drivers/WbDriver2.java @@ -16,41 +16,45 @@ */ package internal.web.drivers; -import be.nbb.sdmx.facade.DataQuery; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.LanguagePriorityList; import be.nbb.sdmx.facade.parser.DataFactory; import be.nbb.sdmx.facade.util.SdmxFix; +import static be.nbb.sdmx.facade.util.SdmxFix.Category.QUERY; import be.nbb.sdmx.facade.web.SdmxWebSource; import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; import internal.web.SdmxWebDriverSupport; import java.io.IOException; import java.net.URL; import be.nbb.sdmx.facade.web.spi.SdmxWebContext; +import internal.web.DataRequest; +import internal.web.SdmxWebClient; +import org.openide.util.lookup.ServiceProvider; /** * * @author Philippe Charles */ +@ServiceProvider(service = SdmxWebDriver.class) public final class WbDriver2 implements SdmxWebDriver { @lombok.experimental.Delegate private final SdmxWebDriverSupport support = SdmxWebDriverSupport .builder() - .name("wb@facade") + .name("web-ri:wb") + .rank(NATIVE_RANK) .client(WbClient2::new) - .supportedProperties(Util.CONNECTION_PROPERTIES) + .supportedProperties(RestClients.CONNECTION_PROPERTIES) .sourceOf("WB", "World Bank", "https://api.worldbank.org/v2/sdmx/rest") .build(); - private static final class WbClient2 extends AbstractSdmx21 { + private static final class WbClient2 extends Sdmx21RestClient { - private WbClient2(SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) { - super(s.getEndpoint(), l, Util.getRestClient(s, c), true, DataFactory.sdmx21()); + private WbClient2(SdmxWebSource s, SdmxWebContext c) { + super(SdmxWebClient.getClientName(s), s.getEndpoint(), c.getLanguages(), RestClients.getRestClient(s, c), true, DataFactory.sdmx21()); } - @SdmxFix(id = "WB#1", cause = "'/' separator required at the end of query") + @SdmxFix(id = 1, category = QUERY, cause = "'/' separator required at the end of query") private static final String SEP = ""; @Override @@ -69,8 +73,8 @@ protected URL getStructureQuery(DataStructureRef ref) throws IOException { } @Override - protected URL getDataQuery(DataflowRef flowRef, DataQuery query) throws IOException { - return getDataQuery(endpoint, flowRef, query).path(SEP).build(); + protected URL getDataQuery(DataRequest request) throws IOException { + return getDataQuery(endpoint, request).path(SEP).build(); } } } diff --git a/sdmx-facade/sdmx-facade-web/src/main/resources/internal/web/drivers/sdmx21.xml b/sdmx-facade/sdmx-facade-web-ri/src/main/resources/internal/web/drivers/sdmx21.xml similarity index 86% rename from sdmx-facade/sdmx-facade-web/src/main/resources/internal/web/drivers/sdmx21.xml rename to sdmx-facade/sdmx-facade-web-ri/src/main/resources/internal/web/drivers/sdmx21.xml index 364c1f8e..960978b6 100644 --- a/sdmx-facade/sdmx-facade-web/src/main/resources/internal/web/drivers/sdmx21.xml +++ b/sdmx-facade/sdmx-facade-web-ri/src/main/resources/internal/web/drivers/sdmx21.xml @@ -3,40 +3,40 @@ ECB European Central Bank - sdmx21@facade + web-ri:sdmx21 https://sdw-wsrest.ecb.europa.eu/service INEGI Instituto Nacional de Estadistica y Geografia - sdmx21@facade + web-ri:sdmx21 http://sdmx.snieg.mx/service/Rest UNDATA Data access system to UN databases - sdmx21@facade + web-ri:sdmx21 http://data.un.org/WS/rest ISTAT Istituto Nazionale di Statistica - sdmx21@facade + web-ri:sdmx21 http://sdmx.istat.it/SDMXWS/rest WITS World Integrated Trade Solutions - sdmx21@facade + web-ri:sdmx21 http://wits.worldbank.org/API/V1/SDMX/V21/rest IMF_SDMX_CENTRAL International Monetary Fund SDMX Central - sdmx21@facade + web-ri:sdmx21 https://sdmxcentral.imf.org/ws/public/sdmxapi/rest diff --git a/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/CacheAssertions.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/CacheAssertions.java new file mode 100644 index 00000000..e1e26f64 --- /dev/null +++ b/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/CacheAssertions.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package _test; + +import be.nbb.sdmx.facade.util.HasCache; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import static org.assertj.core.api.Assertions.*; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class CacheAssertions { + + public void assertCacheBehavior(HasCache o) { + assertThat(o.getCache()).isNotNull(); + ConcurrentMap cache = new ConcurrentHashMap(); + o.setCache(cache); + assertThat(o.getCache()).isSameAs(cache); + o.setCache(null); + assertThat(o.getCache()).isNotNull().isNotSameAs(cache); + } +} diff --git a/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/DriverAssertions.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/DriverAssertions.java new file mode 100644 index 00000000..b719524a --- /dev/null +++ b/sdmx-facade/sdmx-facade-web-ri/src/test/java/_test/DriverAssertions.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package _test; + +import be.nbb.sdmx.facade.util.HasCache; +import be.nbb.sdmx.facade.web.SdmxWebSource; +import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; +import static org.assertj.core.api.Assertions.*; +import be.nbb.sdmx.facade.web.spi.SdmxWebContext; + +/** + * + * @author Philippe Charles + */ +@lombok.experimental.UtilityClass +public class DriverAssertions { + + @SuppressWarnings("null") + public void assertDriverCompliance(SdmxWebDriver d) { + if (d instanceof HasCache) { + CacheAssertions.assertCacheBehavior((HasCache) d); + } + + SdmxWebSource validSource = SdmxWebSource + .builder() + .name("valid") + .driver(d.getName()) + .endpointOf("http://localhost") + .build(); + + SdmxWebSource invalidSource = validSource.toBuilder().driver("").build(); + + SdmxWebContext context = SdmxWebContext.builder().build(); + + assertThat(d.getName()).isNotBlank(); + + assertThatNullPointerException().isThrownBy(() -> d.connect(null, context)); + assertThatNullPointerException().isThrownBy(() -> d.connect(validSource, null)); + + assertThatIllegalArgumentException().isThrownBy(() -> d.connect(invalidSource, context)); + + assertThat(d.getDefaultSources()).allSatisfy(o -> checkSource(o, d)); + + assertThat(d.getClass()).isFinal(); + } + + private void checkSource(SdmxWebSource o, SdmxWebDriver d) { + assertThat(o.getName()).isNotBlank(); + assertThat(o.getDescription()).isNotBlank(); + assertThat(o.getProperties()).isNotNull(); + assertThat(o.getDriver()).isEqualTo(d.getName()); + assertThat(o.getProperties().keySet()).isSubsetOf(d.getSupportedProperties()); + } +} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/rest/RestClientImplTest.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/util/rest/RestClientImplTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/util/rest/RestClientImplTest.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/util/rest/RestClientImplTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/util/rest/RestQueryBuilderTest.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/util/rest/RestQueryBuilderTest.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/util/rest/RestQueryBuilderTest.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/util/rest/RestQueryBuilderTest.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/AbsDriver2Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/AbsDriver2Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/AbsDriver2Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/AbsDriver2Test.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/DtoStatDriver2Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/DotStatDriver2Test.java similarity index 96% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/DtoStatDriver2Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/DotStatDriver2Test.java index 75f8434d..9aa2c378 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/DtoStatDriver2Test.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/DotStatDriver2Test.java @@ -23,7 +23,7 @@ * * @author Philippe Charles */ -public class DtoStatDriver2Test { +public class DotStatDriver2Test { @Test public void testCompliance() { diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/NbbDriver2Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/NbbDriver2Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/NbbDriver2Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/NbbDriver2Test.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/Sdmx21Driver2Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/Sdmx21Driver2Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/Sdmx21Driver2Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/Sdmx21Driver2Test.java diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/AbstractSdmx21Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/Sdmx21RestClientTest.java similarity index 86% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/AbstractSdmx21Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/Sdmx21RestClientTest.java index 778ffec4..a343e2cb 100644 --- a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/AbstractSdmx21Test.java +++ b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/Sdmx21RestClientTest.java @@ -16,11 +16,12 @@ */ package internal.web.drivers; -import be.nbb.sdmx.facade.DataQuery; +import be.nbb.sdmx.facade.DataFilter; import be.nbb.sdmx.facade.DataStructureRef; import be.nbb.sdmx.facade.DataflowRef; import be.nbb.sdmx.facade.Key; -import static internal.web.drivers.AbstractSdmx21.*; +import internal.web.DataRequest; +import static internal.web.drivers.Sdmx21RestClient.*; import java.io.IOException; import java.net.URL; import static org.assertj.core.api.Assertions.*; @@ -30,7 +31,7 @@ * * @author Philippe Charles */ -public class AbstractSdmx21Test { +public class Sdmx21RestClientTest { @Test @SuppressWarnings("null") @@ -78,14 +79,16 @@ public void testGetStructureQuery() throws IOException { public void testGetDataQuery() throws IOException { URL endpoint = new URL("http://localhost"); - assertThatNullPointerException().isThrownBy(() -> getDataQuery(null, specificFlow, DataQuery.of(Key.ALL, true))); - assertThatNullPointerException().isThrownBy(() -> getDataQuery(endpoint, null, DataQuery.of(Key.ALL, true))); - assertThatNullPointerException().isThrownBy(() -> getDataQuery(endpoint, specificFlow, null)); + DataRequest specificRequest = new DataRequest(specificFlow, Key.ALL, DataFilter.SERIES_KEYS_ONLY); + DataRequest genericRequest = new DataRequest(genericFlow, Key.ALL, DataFilter.SERIES_KEYS_ONLY); - assertThat(getDataQuery(endpoint, specificFlow, DataQuery.of(Key.ALL, true)).build()) + assertThatNullPointerException().isThrownBy(() -> getDataQuery(null, specificRequest)); + assertThatNullPointerException().isThrownBy(() -> getDataQuery(endpoint, null)); + + assertThat(getDataQuery(endpoint, specificRequest).build()) .hasToString("http://localhost/data/ECB%2CEXR%2C1.0/all/all?detail=serieskeysonly"); - assertThat(getDataQuery(endpoint, genericFlow, DataQuery.of(Key.ALL, true)).build()) + assertThat(getDataQuery(endpoint, genericRequest).build()) .hasToString("http://localhost/data/all%2CEXR%2Clatest/all/all?detail=serieskeysonly"); } diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/WbDriver2Test.java b/sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/WbDriver2Test.java similarity index 100% rename from sdmx-facade/sdmx-facade-web/src/test/java/internal/web/drivers/WbDriver2Test.java rename to sdmx-facade/sdmx-facade-web-ri/src/test/java/internal/web/drivers/WbDriver2Test.java diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java deleted file mode 100644 index 98f6638d..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/connectors/HasDataCursor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2016 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.connectors; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.Key; -import it.bancaditalia.oss.sdmx.exceptions.SdmxException; -import java.io.IOException; -import javax.annotation.Nonnull; - -/** - * - * @author Philippe Charles - */ -public interface HasDataCursor { - - @Nonnull - DataCursor getDataCursor(@Nonnull DataflowRef flowRef, @Nonnull DataStructure dsd, @Nonnull Key resource, boolean serieskeysonly) throws SdmxException, IOException; -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java deleted file mode 100644 index ba13ded4..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/AbstractXMLStreamReader.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2002-2017 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package internal.org.springframework.util.xml; - -import javax.xml.namespace.QName; -import javax.xml.stream.XMLStreamConstants; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; - -/** - * Abstract base class for {@code XMLStreamReader}s. - * - * @author Arjen Poutsma - * @since 3.0 - */ -abstract class AbstractXMLStreamReader implements XMLStreamReader { - - @Override - public String getElementText() throws XMLStreamException { - if (getEventType() != XMLStreamConstants.START_ELEMENT) { - throw new XMLStreamException("parser must be on START_ELEMENT to read next text", getLocation()); - } - int eventType = next(); - StringBuilder builder = new StringBuilder(); - while (eventType != XMLStreamConstants.END_ELEMENT) { - if (eventType == XMLStreamConstants.CHARACTERS || eventType == XMLStreamConstants.CDATA - || eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.ENTITY_REFERENCE) { - builder.append(getText()); - } else if (eventType == XMLStreamConstants.PROCESSING_INSTRUCTION - || eventType == XMLStreamConstants.COMMENT) { - // skipping - } else if (eventType == XMLStreamConstants.END_DOCUMENT) { - throw new XMLStreamException("unexpected end of document when reading element text content", - getLocation()); - } else if (eventType == XMLStreamConstants.START_ELEMENT) { - throw new XMLStreamException("element text content may not contain START_ELEMENT", getLocation()); - } else { - throw new XMLStreamException("Unexpected event type " + eventType, getLocation()); - } - eventType = next(); - } - return builder.toString(); - } - - @Override - public String getAttributeLocalName(int index) { - return getAttributeName(index).getLocalPart(); - } - - @Override - public String getAttributeNamespace(int index) { - return getAttributeName(index).getNamespaceURI(); - } - - @Override - public String getAttributePrefix(int index) { - return getAttributeName(index).getPrefix(); - } - - @Override - public String getNamespaceURI() { - int eventType = getEventType(); - if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { - return getName().getNamespaceURI(); - } else { - throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); - } - } - - @Override - public String getNamespaceURI(String prefix) { -// Assert.notNull(prefix, "No prefix given"); - return getNamespaceContext().getNamespaceURI(prefix); - } - - @Override - public boolean hasText() { - int eventType = getEventType(); - return eventType == XMLStreamConstants.SPACE || eventType == XMLStreamConstants.CHARACTERS - || eventType == XMLStreamConstants.COMMENT || eventType == XMLStreamConstants.CDATA - || eventType == XMLStreamConstants.ENTITY_REFERENCE; - } - - @Override - public String getPrefix() { - int eventType = getEventType(); - if (eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT) { - return getName().getPrefix(); - } else { - throw new IllegalStateException("parser must be on START_ELEMENT or END_ELEMENT state"); - } - } - - @Override - public boolean hasName() { - int eventType = getEventType(); - return eventType == XMLStreamConstants.START_ELEMENT || eventType == XMLStreamConstants.END_ELEMENT; - } - - @Override - public boolean isWhiteSpace() { - return getEventType() == XMLStreamConstants.SPACE; - } - - @Override - public boolean isStartElement() { - return getEventType() == XMLStreamConstants.START_ELEMENT; - } - - @Override - public boolean isEndElement() { - return getEventType() == XMLStreamConstants.END_ELEMENT; - } - - @Override - public boolean isCharacters() { - return getEventType() == XMLStreamConstants.CHARACTERS; - } - - @Override - public int nextTag() throws XMLStreamException { - int eventType = next(); - while (eventType == XMLStreamConstants.CHARACTERS && isWhiteSpace() - || eventType == XMLStreamConstants.CDATA && isWhiteSpace() || eventType == XMLStreamConstants.SPACE - || eventType == XMLStreamConstants.PROCESSING_INSTRUCTION || eventType == XMLStreamConstants.COMMENT) { - eventType = next(); - } - if (eventType != XMLStreamConstants.START_ELEMENT && eventType != XMLStreamConstants.END_ELEMENT) { - throw new XMLStreamException("expected start or end tag", getLocation()); - } - return eventType; - } - - @Override - public void require(int expectedType, String namespaceURI, String localName) throws XMLStreamException { - int eventType = getEventType(); - if (eventType != expectedType) { - throw new XMLStreamException("Expected [" + expectedType + "] but read [" + eventType + "]"); - } - } - - @Override - @javax.annotation.Nullable - public String getAttributeValue(@javax.annotation.Nullable String namespaceURI, String localName) { - for (int i = 0; i < getAttributeCount(); i++) { - QName name = getAttributeName(i); - if (name.getLocalPart().equals(localName) - && (namespaceURI == null || name.getNamespaceURI().equals(namespaceURI))) { - return getAttributeValue(i); - } - } - return null; - } - - @Override - public boolean hasNext() throws XMLStreamException { - return getEventType() != END_DOCUMENT; - } - - @Override - public String getLocalName() { - return getName().getLocalPart(); - } - - @Override - public char[] getTextCharacters() { - return getText().toCharArray(); - } - - @Override - public int getTextCharacters(int sourceStart, char[] target, int targetStart, int length) - throws XMLStreamException { - char[] source = getTextCharacters(); - length = Math.min(length, source.length); - System.arraycopy(source, sourceStart, target, targetStart, length); - return length; - } - - @Override - public int getTextLength() { - return getText().length(); - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java deleted file mode 100644 index 31ceb04d..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/org/springframework/util/xml/XMLEventStreamReader.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package internal.org.springframework.util.xml; - -import java.util.Iterator; -import javax.xml.namespace.NamespaceContext; -import javax.xml.namespace.QName; -import javax.xml.stream.Location; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.events.Attribute; -import javax.xml.stream.events.Comment; -import javax.xml.stream.events.Namespace; -import javax.xml.stream.events.ProcessingInstruction; -import javax.xml.stream.events.StartDocument; -import javax.xml.stream.events.XMLEvent; - -/** - * Implementation of the {@link javax.xml.stream.XMLStreamReader} interface that - * wraps a {@link XMLEventReader}. Useful because the StAX - * {@link javax.xml.stream.XMLInputFactory} allows one to create a event reader - * from a stream reader, but not vice-versa. - * - * @author Arjen Poutsma - * @since 3.0 - * @see StaxUtils#createEventStreamReader(javax.xml.stream.XMLEventReader) - */ -public final class XMLEventStreamReader extends AbstractXMLStreamReader { - - private XMLEvent event; - - private final XMLEventReader eventReader; - - public XMLEventStreamReader(XMLEventReader eventReader) throws XMLStreamException { - this.eventReader = eventReader; - this.event = eventReader.nextEvent(); - } - - @Override - public QName getName() { - if (this.event.isStartElement()) { - return this.event.asStartElement().getName(); - } else if (this.event.isEndElement()) { - return this.event.asEndElement().getName(); - } else { - throw new IllegalStateException(); - } - } - - @Override - public Location getLocation() { - return this.event.getLocation(); - } - - @Override - public int getEventType() { - return this.event.getEventType(); - } - - @Override - @javax.annotation.Nullable - public String getVersion() { - if (this.event.isStartDocument()) { - return ((StartDocument) this.event).getVersion(); - } else { - return null; - } - } - - @Override - public Object getProperty(String name) throws IllegalArgumentException { - return this.eventReader.getProperty(name); - } - - @Override - public boolean isStandalone() { - if (this.event.isStartDocument()) { - return ((StartDocument) event).isStandalone(); - } else { - throw new IllegalStateException(); - } - } - - @Override - public boolean standaloneSet() { - if (this.event.isStartDocument()) { - return ((StartDocument) this.event).standaloneSet(); - } else { - throw new IllegalStateException(); - } - } - - @Override - @javax.annotation.Nullable - public String getEncoding() { - return null; - } - - @Override - @javax.annotation.Nullable - public String getCharacterEncodingScheme() { - return null; - } - - @Override - public String getPITarget() { - if (this.event.isProcessingInstruction()) { - return ((ProcessingInstruction) this.event).getTarget(); - } else { - throw new IllegalStateException(); - } - } - - @Override - public String getPIData() { - if (this.event.isProcessingInstruction()) { - return ((ProcessingInstruction) this.event).getData(); - } else { - throw new IllegalStateException(); - } - } - - @Override - public int getTextStart() { - return 0; - } - - @Override - public String getText() { - if (this.event.isCharacters()) { - return event.asCharacters().getData(); - } else if (this.event.getEventType() == XMLEvent.COMMENT) { - return ((Comment) this.event).getText(); - } else { - throw new IllegalStateException(); - } - } - - @Override - @SuppressWarnings("rawtypes") - public int getAttributeCount() { - if (!this.event.isStartElement()) { - throw new IllegalStateException(); - } - Iterator attributes = this.event.asStartElement().getAttributes(); - return countIterator(attributes); - } - - @Override - public boolean isAttributeSpecified(int index) { - return getAttribute(index).isSpecified(); - } - - @Override - public QName getAttributeName(int index) { - return getAttribute(index).getName(); - } - - @Override - public String getAttributeType(int index) { - return getAttribute(index).getDTDType(); - } - - @Override - public String getAttributeValue(int index) { - return getAttribute(index).getValue(); - } - - @SuppressWarnings("rawtypes") - private Attribute getAttribute(int index) { - if (!this.event.isStartElement()) { - throw new IllegalStateException(); - } - int count = 0; - Iterator attributes = this.event.asStartElement().getAttributes(); - while (attributes.hasNext()) { - Attribute attribute = (Attribute) attributes.next(); - if (count == index) { - return attribute; - } else { - count++; - } - } - throw new IllegalArgumentException(); - } - - @Override - public NamespaceContext getNamespaceContext() { - if (this.event.isStartElement()) { - return this.event.asStartElement().getNamespaceContext(); - } else { - throw new IllegalStateException(); - } - } - - @Override - @SuppressWarnings("rawtypes") - public int getNamespaceCount() { - Iterator namespaces; - if (this.event.isStartElement()) { - namespaces = this.event.asStartElement().getNamespaces(); - } else if (this.event.isEndElement()) { - namespaces = this.event.asEndElement().getNamespaces(); - } else { - throw new IllegalStateException(); - } - return countIterator(namespaces); - } - - @Override - public String getNamespacePrefix(int index) { - return getNamespace(index).getPrefix(); - } - - @Override - public String getNamespaceURI(int index) { - return getNamespace(index).getNamespaceURI(); - } - - @SuppressWarnings("rawtypes") - private Namespace getNamespace(int index) { - Iterator namespaces; - if (this.event.isStartElement()) { - namespaces = this.event.asStartElement().getNamespaces(); - } else if (this.event.isEndElement()) { - namespaces = this.event.asEndElement().getNamespaces(); - } else { - throw new IllegalStateException(); - } - int count = 0; - while (namespaces.hasNext()) { - Namespace namespace = (Namespace) namespaces.next(); - if (count == index) { - return namespace; - } else { - count++; - } - } - throw new IllegalArgumentException(); - } - - @Override - public int next() throws XMLStreamException { - this.event = this.eventReader.nextEvent(); - return this.event.getEventType(); - } - - @Override - public void close() throws XMLStreamException { - this.eventReader.close(); - } - - @SuppressWarnings("rawtypes") - private static int countIterator(Iterator iterator) { - int count = 0; - while (iterator.hasNext()) { - iterator.next(); - count++; - } - return count; - } - -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/SdmxWebResource.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/SdmxWebResource.java deleted file mode 100644 index 386c9de7..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/util/drivers/SdmxWebResource.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2018 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.util.drivers; - -import be.nbb.sdmx.facade.web.SdmxWebSource; -import ioutil.Jaxb; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlRootElement; - -/** - * - * @author Philippe Charles - */ -@lombok.experimental.UtilityClass -public final class SdmxWebResource { - - public static List load(String resource) { - try { - return Jaxb.Parser.of(XSources.class) - .parseStream(() -> SdmxWebResource.class.getResourceAsStream(resource)) - .toSources(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - @XmlRootElement(name = "sources") - public static final class XSources { - - public XSource[] source; - - public List toSources() { - return source != null - ? Stream.of(source).map(XSource::toSource).collect(Collectors.toList()) - : Collections.emptyList(); - } - - public static XSources of(List o) { - XSources result = new XSources(); - result.source = o.stream().map(XSource::of).toArray(XSource[]::new); - return result; - } - } - - public static final class XSource { - - public String name; - - public String description; - - public String driver; - - public String endpoint; - - public XProperty[] property; - - public SdmxWebSource toSource() { - SdmxWebSource.Builder result = SdmxWebSource - .builder() - .name(name) - .description(description) - .driver(driver) - .endpointOf(endpoint); - if (property != null) { - for (XProperty o : property) { - result.property(o.key, o.value); - } - } - return result.build(); - } - - public static XSource of(SdmxWebSource o) { - XSource result = new XSource(); - result.name = o.getName(); - result.description = o.getDescription(); - result.driver = o.getDriver(); - result.endpoint = o.getEndpoint().toString(); - result.property = o.getProperties().entrySet().stream().map(XProperty::of).toArray(XProperty[]::new); - return result; - } - } - - public static final class XProperty { - - @XmlAttribute - String key; - - @XmlAttribute - String value; - - public static XProperty of(Map.Entry o) { - XProperty result = new XProperty(); - result.key = o.getKey(); - result.value = o.getValue(); - return result; - } - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/NbbDriver2.java b/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/NbbDriver2.java deleted file mode 100644 index ac1c1791..00000000 --- a/sdmx-facade/sdmx-facade-web/src/main/java/internal/web/drivers/NbbDriver2.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package internal.web.drivers; - -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataflowRef; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.util.SdmxFix; -import be.nbb.sdmx.facade.web.SdmxWebSource; -import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import internal.util.rest.RestQueryBuilder; -import internal.web.SdmxWebDriverSupport; -import java.io.IOException; -import java.net.URL; -import be.nbb.sdmx.facade.web.spi.SdmxWebContext; - -/** - * - * @author Philippe Charles - */ -public final class NbbDriver2 implements SdmxWebDriver { - - @lombok.experimental.Delegate - private final SdmxWebDriverSupport support = SdmxWebDriverSupport - .builder() - .name("nbb@facade") - .client(NbbClient2::new) - .supportedProperties(Util.CONNECTION_PROPERTIES) - .sourceOf("NBB", "National Bank Belgium", "https://stat.nbb.be/restsdmx/sdmx.ashx") - .build(); - - private static final class NbbClient2 extends AbstractDotStat { - - private NbbClient2(SdmxWebSource s, LanguagePriorityList l, SdmxWebContext c) { - super(s.getEndpoint(), l, Util.getRestClient(s, c)); - } - - @SdmxFix(id = "NBB#1", cause = "'/all' must be encoded to '%2Fall'") - @Override - protected URL getDataQuery(DataflowRef flowRef, DataQuery query) throws IOException { - return RestQueryBuilder.of(endpoint) - .path(DATA_RESOURCE) - .path(flowRef.getId()) - .path(query.getKey().toString() + "/all") - .param("format", "compact_v2") - .build(); - } - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/FailingWebClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/FailingWebClient.java deleted file mode 100644 index b1c74925..00000000 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/FailingWebClient.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package _test.client; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; -import java.io.IOException; -import java.util.List; -import java.time.Duration; -import internal.web.SdmxWebClient; - -/** - * - * @author Philippe Charles - */ -public enum FailingWebClient implements SdmxWebClient { - - INSTANCE; - - @Override - public List getFlows() throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public Dataflow getFlow(DataflowRef ref) throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public DataStructure getStructure(DataStructureRef ref) throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public boolean isSeriesKeysOnlySupported() throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public Duration ping() throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NoOpWebClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NoOpWebClient.java deleted file mode 100644 index 0ae071f9..00000000 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NoOpWebClient.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package _test.client; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.time.Duration; -import internal.web.SdmxWebClient; - -/** - * - * @author Philippe Charles - */ -public enum NoOpWebClient implements SdmxWebClient { - - INSTANCE; - - @Override - public List getFlows() throws IOException { - throw new IOException(); - } - - @Override - public Dataflow getFlow(DataflowRef ref) throws IOException { - Objects.requireNonNull(ref); - throw new IOException(); - } - - @Override - public DataStructure getStructure(DataStructureRef ref) throws IOException { - Objects.requireNonNull(ref); - throw new IOException(); - } - - @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - Objects.requireNonNull(flowRef); - Objects.requireNonNull(dsd); - Objects.requireNonNull(query); - throw new IOException(); - } - - @Override - public boolean isSeriesKeysOnlySupported() throws IOException { - throw new IOException(); - } - - @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { - Objects.requireNonNull(flowRef); - throw new IOException(); - } - - @Override - public Duration ping() throws IOException { - throw new IOException(); - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NullWebClient.java b/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NullWebClient.java deleted file mode 100644 index 8dd95054..00000000 --- a/sdmx-facade/sdmx-facade-web/src/test/java/_test/client/NullWebClient.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package _test.client; - -import be.nbb.sdmx.facade.DataCursor; -import be.nbb.sdmx.facade.DataQuery; -import be.nbb.sdmx.facade.DataStructure; -import be.nbb.sdmx.facade.DataStructureRef; -import be.nbb.sdmx.facade.Dataflow; -import be.nbb.sdmx.facade.DataflowRef; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.time.Duration; -import internal.web.SdmxWebClient; - -/** - * - * @author Philippe Charles - */ -public enum NullWebClient implements SdmxWebClient { - - INSTANCE; - - @Override - public List getFlows() throws IOException { - return null; - } - - @Override - public Dataflow getFlow(DataflowRef ref) throws IOException { - Objects.requireNonNull(ref); - return null; - } - - @Override - public DataStructure getStructure(DataStructureRef ref) throws IOException { - Objects.requireNonNull(ref); - return null; - } - - @Override - public DataCursor getData(DataflowRef flowRef, DataQuery query, DataStructure dsd) throws IOException { - Objects.requireNonNull(flowRef); - Objects.requireNonNull(dsd); - Objects.requireNonNull(query); - return null; - } - - @Override - public boolean isSeriesKeysOnlySupported() throws IOException { - return false; - } - - @Override - public DataStructureRef peekStructureRef(DataflowRef flowRef) throws IOException { - Objects.requireNonNull(flowRef); - return null; - } - - @Override - public Duration ping() throws IOException { - return null; - } -} diff --git a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java b/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java deleted file mode 100644 index 5a885add..00000000 --- a/sdmx-facade/sdmx-facade-web/src/test/java/be/nbb/sdmx/facade/web/SdmxWebManagerTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2017 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or - as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package be.nbb.sdmx.facade.web; - -import be.nbb.sdmx.facade.web.spi.SdmxWebContext; -import be.nbb.sdmx.facade.LanguagePriorityList; -import be.nbb.sdmx.facade.SdmxConnection; -import be.nbb.sdmx.facade.repo.SdmxRepository; -import be.nbb.sdmx.facade.tck.ConnectionSupplierAssert; -import be.nbb.sdmx.facade.web.spi.SdmxWebDriver; -import java.io.IOException; -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import static org.assertj.core.api.Assertions.*; -import org.junit.Test; - -/** - * - * @author Philippe Charles - */ -public class SdmxWebManagerTest { - - @Test - public void testCompliance() { - ConnectionSupplierAssert.assertCompliance(SdmxWebManager.of(REPO), HELLO.getName(), "ko"); - } - - @Test - @SuppressWarnings("null") - public void testFactories() { - assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((Iterable) null)); - assertThatNullPointerException().isThrownBy(() -> SdmxWebManager.of((SdmxWebDriver[]) null)); - } - - @Test - @SuppressWarnings("null") - public void testGetConnectionOfSource() { - SdmxWebManager manager = SdmxWebManager.of(REPO); - assertThatNullPointerException().isThrownBy(() -> manager.getConnection((SdmxWebSource) null)); - assertThatIOException().isThrownBy(() -> manager.getConnection(HELLO.toBuilder().endpointOf("http://ko").build())); - } - - private static final SdmxWebSource HELLO = SdmxWebSource.builder().name("ok").driver(RepoDriver.NAME).endpointOf("http://r1").build(); - private static final SdmxWebDriver REPO = new RepoDriver(); - - private static final class RepoDriver implements SdmxWebDriver { - - static final String NAME = "repo"; - - final List repos = Collections.singletonList(SdmxRepository.builder().name("http://r1").build()); - - @Override - public String getName() { - return NAME; - } - - @Override - public SdmxWebConnection connect(SdmxWebSource source, LanguagePriorityList langs, SdmxWebContext context) throws IOException { - return repos.stream() - .filter(o -> o.getName().equals(source.getEndpoint().toString())) - .findFirst() - .map(o -> new RepoWebConnection(o.asConnection())) - .orElseThrow(IOException::new); - } - - @Override - public Collection getDefaultSources() { - return Collections.singletonList(HELLO); - } - - @Override - public Collection getSupportedProperties() { - return Collections.emptyList(); - } - } - - @lombok.AllArgsConstructor - private static final class RepoWebConnection implements SdmxWebConnection { - - @lombok.experimental.Delegate - private final SdmxConnection delegate; - - @Override - public Duration ping() throws IOException { - return Duration.ZERO; - } - } -}