Merge branch 'release-2.2.3'

nbbrd · Jan 14, 2019 · 1806bcb · 1806bcb
2 parents dc9fc01 + df2aa0e
commit 1806bcb
Show file tree

Hide file tree

Showing 147 changed files with 2,240 additions and 1,797 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,6 @@
 /sdmx-facade/sdmx-facade-util/target/
 /sdmx-facade/sdmx-facade-file/target/
 /sdmx-facade/sdmx-facade-samples/target/
-/sdmx-facade/sdmx-facade-web/target/
+/sdmx-facade/sdmx-facade-web-ri/target/
+/sdmx-facade/sdmx-facade-util-web/target/
+/sdmx-facade/sdmx-facade-util-xml/target/
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,26 @@
 language: java
 jdk:
   - oraclejdk8
+  - openjdk11
+
+env:
+  global:
+    - DEPLOY_JDK=oraclejdk8
+    - DEPLOY_REPO=nbbrd/jdemetra-dotstat
+    - DEPLOY_BIN=demetra-dotstat-desktop/target/*.nbm
+
+deploy:
+  # Github releases from tags
+  - provider: releases
+    api_key: "${GITHUB_KEY}"
+    skip_cleanup: true
+    draft: true
+    file_glob: true
+    file: "${DEPLOY_BIN}"
+    on:
+      tags: true
+      repo: "${DEPLOY_REPO}"
+      condition: $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_JDK_VERSION == $DEPLOY_JDK
 
 cache:
   directories:

diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 # SDMX extension for JDemetra+
 
-This plugin provides time series from [SDMX](https://sdmx.org/) to [JDemetra+](https://github.com/jdemetra/jdemetra-app) by using [SDMX Web Services](https://github.com/sdmx-twg/sdmx-rest/wiki).
-
 [![Download](https://img.shields.io/github/release/nbbrd/jdemetra-dotstat.svg)](https://github.com/nbbrd/jdemetra-dotstat/releases/latest)
 
-## Quickstart
+This plugin provides time series from [SDMX](https://sdmx.org/) to [JDemetra+](https://github.com/jdemetra/jdemetra-app) by querying [web services](https://github.com/nbbrd/jdemetra-dotstat/wiki/Supported-web-services) or parsing [files](https://github.com/nbbrd/jdemetra-dotstat/wiki/Supported-file-formats).
+
+![Browsing web service](https://github.com/nbbrd/jdemetra-dotstat/wiki/assets/browse_web_service.gif)
 
 See [documentation](https://github.com/nbbrd/jdemetra-dotstat/wiki).
diff --git a/demetra-dotstat-core/pom.xml b/demetra-dotstat-core/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>be.nbb.demetra</groupId>
     <artifactId>demetra-dotstat-core</artifactId>
-    <version>2.2.2</version>
+    <version>2.2.3</version>
     <packaging>jar</packaging>
 
     <name>Demetra - DotStat - Core</name>
@@ -27,8 +27,11 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.target>1.8</maven.compiler.target>
         <maven.compiler.source>1.8</maven.compiler.source>
+        <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
+
         <!-- dependencies -->
         <demetra.version>2.2.0</demetra.version>
+        <lombok.version>1.18.4</lombok.version>
     </properties>
 
     <dependencyManagement>
@@ -40,6 +43,11 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>org.projectlombok</groupId>
+                <artifactId>lombok</artifactId>
+                <version>${lombok.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -73,17 +81,23 @@
         </dependency>
         <dependency>
             <groupId>be.nbb.sdmx</groupId>
-            <artifactId>sdmx-facade-web</artifactId>
+            <artifactId>sdmx-facade-file</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>eu.europa.ec.joinup.sat</groupId>
+            <artifactId>demetra-jdbc</artifactId>
+        </dependency>
+        <!-- runtime only -->
         <dependency>
             <groupId>be.nbb.sdmx</groupId>
-            <artifactId>sdmx-facade-file</artifactId>
+            <artifactId>sdmx-facade-connectors</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>eu.europa.ec.joinup.sat</groupId>
-            <artifactId>demetra-jdbc</artifactId>
+            <groupId>be.nbb.sdmx</groupId>
+            <artifactId>sdmx-facade-web-ri</artifactId>
+            <version>${project.version}</version>
         </dependency>
         <!-- test only -->
         <dependency>
@@ -109,6 +123,41 @@
         </dependency>
     </dependencies>
 
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <version>${maven-compiler-plugin.version}</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
+    <profiles>
+        <!-- Add missing Java EE Modules on JDK11 -->
+        <profile>
+            <activation>
+                <jdk>11</jdk>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
+                    <version>2.4.0-b180830.0359</version>
+                    <scope>provided</scope>
+                </dependency>
+                <dependency>
+                    <groupId>org.glassfish.jaxb</groupId>
+                    <artifactId>jaxb-runtime</artifactId>
+                    <version>2.4.0-b180830.0438</version>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+
     <repositories>
         <repository>
             <id>netbeans-releases</id>
@@ -118,5 +167,12 @@
             <id>oss-jfrog-artifactory-releases</id>
             <url>https://oss.jfrog.org/artifactory/oss-release-local</url>
         </repository>
+        <repository>
+            <id>projectlombok.org</id>
+            <url>https://projectlombok.org/edge-releases</url>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
     </repositories>
 </project>
diff --git a/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java b/demetra-dotstat-core/src/main/java/be/nbb/demetra/sdmx/file/SdmxFileProvider.java
@@ -49,6 +49,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import be.nbb.sdmx.facade.SdmxManager;
+import ec.tss.TsAsyncMode;
+import ec.tss.tsproviders.cursor.TsCursorAsFiller;
+import ec.tss.tsproviders.utils.TsFillerAsProvider;
+import java.io.EOFException;
 
 /**
  *
@@ -78,6 +82,9 @@ public final class SdmxFileProvider implements IFileLoader, HasSdmxProperties {
     @lombok.experimental.Delegate(excludes = {HasTsCursor.class, HasDataDisplayName.class})
     private final CubeSupport cubeSupport;
 
+    @lombok.experimental.Delegate
+    private final HasDataDisplayName dataDisplayName;
+
     @lombok.experimental.Delegate
     private final ITsProvider tsSupport;
 
@@ -92,7 +99,8 @@ public SdmxFileProvider() {
         this.beanSupport = HasDataSourceBean.of(NAME, sdmxParam, sdmxParam.getVersion());
         this.filePathSupport = HasFilePaths.of(cache::invalidateAll);
         this.cubeSupport = CubeSupport.of(new SdmxCubeResource(cache, properties, filePathSupport, sdmxParam));
-        this.tsSupport = CubeSupport.asTsProvider(NAME, logger, cubeSupport, monikerSupport, cache::invalidateAll);
+        this.dataDisplayName = new SdmxFileDataDisplayName(beanSupport, cubeSupport);
+        this.tsSupport = TsFillerAsProvider.of(NAME, TsAsyncMode.Once, TsCursorAsFiller.of(logger, cubeSupport, monikerSupport, dataDisplayName), cache::invalidateAll);
     }
 
     @Override
@@ -110,24 +118,8 @@ public boolean accept(File pathname) {
         return pathname.getName().toLowerCase().endsWith(".xml");
     }
 
-    @Override
-    public String getDisplayName(DataSource dataSource) throws IllegalArgumentException {
-        return getSourceLabel(decodeBean(dataSource));
-    }
-
-    @Override
-    public String getDisplayName(DataSet dataSet) throws IllegalArgumentException {
-        return cubeSupport.getDisplayName(dataSet);
-    }
-
-    @Override
-    public String getDisplayName(IOException exception) throws IllegalArgumentException {
-        return cubeSupport.getDisplayName(exception);
-    }
-
-    @Override
-    public String getDisplayNodeName(DataSet dataSet) throws IllegalArgumentException {
-        return cubeSupport.getDisplayNodeName(dataSet);
+    private static String getSourceLabel(SdmxFileBean bean) {
+        return bean.getFile().getPath();
     }
 
     @lombok.AllArgsConstructor
@@ -182,7 +174,33 @@ private static IO.Supplier<SdmxConnection> toConnection(HasSdmxProperties proper
         }
     }
 
-    private static String getSourceLabel(SdmxFileBean bean) {
-        return bean.getFile().getPath();
+    @lombok.AllArgsConstructor
+    private static final class SdmxFileDataDisplayName implements HasDataDisplayName {
+
+        private final HasDataSourceBean<SdmxFileBean> beanSupport;
+        private final CubeSupport cubeSupport;
+
+        @Override
+        public String getDisplayName(DataSource dataSource) throws IllegalArgumentException {
+            return getSourceLabel(beanSupport.decodeBean(dataSource));
+        }
+
+        @Override
+        public String getDisplayName(DataSet dataSet) throws IllegalArgumentException {
+            return cubeSupport.getDisplayName(dataSet);
+        }
+
+        @Override
+        public String getDisplayName(IOException exception) throws IllegalArgumentException {
+            if (exception instanceof EOFException) {
+                return "Unexpected end-of-file: " + exception.getMessage();
+            }
+            return cubeSupport.getDisplayName(exception);
+        }
+
+        @Override
+        public String getDisplayNodeName(DataSet dataSet) throws IllegalArgumentException {
+            return cubeSupport.getDisplayNodeName(dataSet);
+        }
     }
 }
diff --git a/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java b/demetra-dotstat-core/src/main/java/internal/sdmx/SdmxQueryUtil.java
@@ -20,7 +20,7 @@
 import be.nbb.sdmx.facade.DataflowRef;
 import be.nbb.sdmx.facade.Dimension;
 import be.nbb.sdmx.facade.Key;
-import be.nbb.sdmx.facade.DataQuery;
+import be.nbb.sdmx.facade.DataFilter;
 import be.nbb.sdmx.facade.SdmxConnection;
 import com.google.common.collect.ImmutableList;
 import ec.tss.tsproviders.cursor.TsCursor;
@@ -85,7 +85,7 @@ public List<String> getChildren(SdmxConnection conn, DataflowRef flowRef, Key re
 
     //<editor-fold defaultstate="collapsed" desc="Implementation details">
     private TsCursor<Key> request(SdmxConnection conn, DataflowRef flowRef, Key key, String labelAttribute, boolean seriesKeysOnly) throws IOException {
-        return new SdmxDataAdapter(key, conn.getCursor(flowRef, DataQuery.of(key, seriesKeysOnly)), labelAttribute);
+        return new SdmxDataAdapter(key, conn.getDataCursor(flowRef, key, seriesKeysOnly ? DataFilter.SERIES_KEYS_ONLY : DataFilter.ALL), labelAttribute);
     }
 
     private TsCursor<Key> computeKeys(SdmxConnection conn, DataflowRef flowRef, Key key) throws IOException {

diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/dotstat/DotStatAccessorTest.java
@@ -20,7 +20,7 @@
 import be.nbb.sdmx.facade.DataStructure;
 import be.nbb.sdmx.facade.Dimension;
 import be.nbb.sdmx.facade.Key;
-import be.nbb.sdmx.facade.DataQuery;
+import be.nbb.sdmx.facade.DataFilter;
 import be.nbb.sdmx.facade.Series;
 import be.nbb.sdmx.facade.repo.SdmxRepositoryManager;
 import com.google.common.base.Joiner;
@@ -108,7 +108,7 @@ public void testGetKey() throws Exception {
     @Test
     public void testGetKeyFromTs() throws Exception {
         assertThat(manager.getConnection("NBB")
-                .getStream(NBB_FLOW_REF, DataQuery.of(Key.ALL, true))
+                .getDataStream(NBB_FLOW_REF, Key.ALL, DataFilter.SERIES_KEYS_ONLY)
                 .map(Series::getKey)).contains(Key.parse("LOCSTL04.AUS.M"));
     }
 

diff --git a/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java b/demetra-dotstat-core/src/test/java/be/nbb/demetra/sdmx/file/SdmxFileProviderTest.java
@@ -146,12 +146,12 @@ public void testContent() throws IOException {
 
             assertThat(newColInfo(p, NO_XML, STRUCT20)).satisfies(info -> {
                 assertThat(p.get(info)).isFalse();
-                assertThat(info.invalidDataCause).contains("XMLStreamException");
+                assertThat(info.invalidDataCause).contains("end-of-file");
             });
 
             assertThat(newColInfo(p, GENERIC20, NO_XML)).satisfies(info -> {
                 assertThat(p.get(info)).isFalse();
-                assertThat(info.invalidDataCause).contains("XMLStreamException");
+                assertThat(info.invalidDataCause).contains("end-of-file");
             });
         }
     }

diff --git a/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java b/demetra-dotstat-core/src/test/java/test/samples/FacadeResource.java
@@ -26,6 +26,7 @@
 import be.nbb.sdmx.facade.samples.SdmxSource;
 import be.nbb.sdmx.facade.repo.SdmxRepository;
 import be.nbb.sdmx.facade.Series;
+import be.nbb.sdmx.facade.parser.DataFactory;
 import be.nbb.sdmx.facade.util.SeriesSupport;
 import be.nbb.sdmx.facade.xml.stream.SdmxXmlStreams;
 import java.io.IOException;
@@ -104,7 +105,7 @@ private List<Dataflow> flow20(ByteSource xml, LanguagePriorityList l) throws IOE
     }
 
     List<Series> data20(ByteSource xml, DataStructure dsd) throws IOException {
-        try (DataCursor c = SdmxXmlStreams.genericData20(dsd).parseReader(xml::openReader)) {
+        try (DataCursor c = SdmxXmlStreams.genericData20(dsd, DataFactory.sdmx20()).parseReader(xml::openReader)) {
             return SeriesSupport.copyOf(c);
         }
     }
@@ -118,7 +119,7 @@ private List<Dataflow> flow21(ByteSource xml, LanguagePriorityList l) throws IOE
     }
 
     List<Series> data21(ByteSource xml, DataStructure dsd) throws IOException {
-        try (DataCursor c = SdmxXmlStreams.genericData21(dsd).parseReader(xml::openReader)) {
+        try (DataCursor c = SdmxXmlStreams.genericData21(dsd, DataFactory.sdmx21()).parseReader(xml::openReader)) {
             return SeriesSupport.copyOf(c);
         }
     }