Merge pull request #210 from kbss-cvut/issues/issue205-module-creator…

…-plugin-outputs-wrong-rdf

Fix issues #205, #206 and #216

s-pipes-core/src/main/java/cz/cvut/spipes/modules/Parameter.java

@@ -11,4 +11,5 @@
 public @interface Parameter {
     String urlPrefix() default KBSS_MODULE.uri;
     String name();
+    String comment();
 }

s-pipes-modules-utils/s-pipes-module-creator-maven-plugin/src/main/java/RdfAnnotationProcessorMojo.java

@@ -139,9 +139,12 @@ private void generateRdfForAllModules() throws MalformedURLException, ClassNotFo
             for (Class<?> moduleClass : moduleClasses) {
                 getLog().info("Creating RDF for module '" + moduleClass.getCanonicalName() + "'");
                 var moduleAnnotation = readModuleAnnotationFromClass(moduleClass);
+                var extendedModuleAnnotation = readExtendedModuleAnnotationFromClass(moduleClass);
                 var constraints = readConstraintsFromClass(moduleClass);
-                writeConstraintsToModel(model, constraints, moduleAnnotation);
+                writeConstraintsToModel(model, constraints, moduleAnnotation, extendedModuleAnnotation);
             }
+            Optional.ofNullable(readManuallyManagedModuleDescriptionOntology(submodule))
+                    .ifPresent(model::add);
 
             getLog().info("--------------------------------------");
         }
@@ -154,14 +157,27 @@ private void generateRdfForAllModules() throws MalformedURLException, ClassNotFo
         getLog().info("======================================");
     }
 
+    private SPipesModule readExtendedModuleAnnotationFromClass(Class<?> moduleClass) {
+
+        SPipesModule ret = null;
+        Class cls = moduleClass.getSuperclass();
+        while(ret == null && cls != null) {
+            ret = readModuleAnnotationFromClass(cls);
+            cls = cls.getSuperclass();
+        }
+
+        return ret;
+    }
+
     private void generateRdfForModule() throws MojoExecutionException {
         try {
             Set<Class<?>> moduleClasses = readAllModuleClasses(this.project);
             var model = readModelFromDefaultFile();
             for (Class<?> moduleClass : moduleClasses) {
                 var moduleAnnotation = readModuleAnnotationFromClass(moduleClass);
+                var extendedModuleAnnotation = readExtendedModuleAnnotationFromClass(moduleClass);
                 var constraints = readConstraintsFromClass(moduleClass);
-                writeConstraintsToModel(model, constraints, moduleAnnotation);
+                writeConstraintsToModel(model, constraints, moduleAnnotation, extendedModuleAnnotation);
             }
 
             var ontologyPath = modulePackageName.replaceAll("[.]", "/") + "/" + ontologyFilename;
@@ -203,10 +219,45 @@ private Set<Class<?>> readAllModuleClasses(MavenProject project) throws Malforme
         return moduleClasses;
     }
 
+    /**
+     * Reads manually managed modules ontology from maven sub-project.
+     * The ontology is loaded from the resource folder of the sub-project and it is expected to start with the artifact
+     * id of the sub-project followed by the postfix ".tll". For example, assume the input parameter project
+     * artifact id "s-pipes-modules-text-analysis". This method will look for the ontology located at:
+     * <pre>   $resource-dir$/s-pipes-modules-text-analysis.ttl</pre>
+     *
+     * Any ontology resources and their triples are removed from the model before returning.
+     * @param project
+     * @return
+     */
+    private Model readManuallyManagedModuleDescriptionOntology(MavenProject project){
+        String ontoName = project.getArtifactId() + ".ttl";
+        Optional<String> ontoUri = Optional.ofNullable(project).map(p ->
+                p.getResources().stream()
+                        .map(r -> new File(r.getDirectory(), ontoName))
+                        .filter(File::exists)
+                        .map(f -> f.getAbsoluteFile().toURI().toString())
+                        .findFirst().orElse(null)
+        );
+
+        if (!ontoUri.isPresent())
+            return null;
+
+        Model m = ModelFactory.createDefaultModel();
+        m.read(ontoUri.get(), "TTL");
+        m.listSubjectsWithProperty(RDF.type, OWL.Ontology).toList()
+                .forEach(o -> m.removeAll(o, null, null));
+
+        return m;
+    }
+
     private URL[] getDependencyURLs(MavenProject project) throws MalformedURLException {
         Set<URL> ret = new HashSet<>();
         ret.add(new File(project.getBuild().getOutputDirectory()).toURI().toURL());
-        for(Dependency d : project.getDependencies()){
+
+        for(Dependency d : project.getDependencies().stream()
+                .filter(d -> !Artifact.SCOPE_TEST.equals(d.getScope()))
+                .collect(Collectors.toList())){
             URL dURL = getURL(getLocalRepository(project), d);
             ret.add(dURL);
         }
@@ -239,10 +290,21 @@ private SPipesModule readModuleAnnotationFromClass(Class<?> classObject) {
     }
 
     private List<cz.cvut.spipes.modules.Parameter> readConstraintsFromClass(Class<?> classObject) {
-        return Arrays.stream(classObject.getDeclaredFields())
-                .filter(field -> field.isAnnotationPresent(PARAM_ANNOTATION))
-                .map(field -> field.getAnnotation(PARAM_ANNOTATION))
-                .collect(Collectors.toUnmodifiableList());
+        List<cz.cvut.spipes.modules.Parameter> parameterConstraints = new ArrayList<>();
+
+        Class cls = classObject;
+        while(cls != null){
+            if(cls != classObject && cls.isAnnotationPresent(MODULE_ANNOTATION)) {
+                cls = cls.getSuperclass();
+                continue;
+            }
+            Arrays.stream(cls.getDeclaredFields())
+                    .filter(field -> field.isAnnotationPresent(PARAM_ANNOTATION))
+                    .map(field -> field.getAnnotation(PARAM_ANNOTATION))
+                    .forEach(parameterConstraints::add);
+            cls = cls.getSuperclass();
+        }
+        return parameterConstraints;
     }
     //endregion
 
@@ -269,25 +331,28 @@ private Model readModelFromDefaultFile() {
 
     private void writeConstraintsToModel(Model baseRdfModel,
                                          List<cz.cvut.spipes.modules.Parameter> constraintAnnotations,
-                                         SPipesModule moduleAnnotation) {
+                                         SPipesModule moduleAnnotation,
+                                         SPipesModule extendedModuleAnnotation) {
         final var root = ResourceFactory.createResource(KBSS_MODULE.uri + moduleAnnotation.label().replaceAll(" ", "-").toLowerCase()); //todo can be added to the annotation
+        // set extended uri
+        Optional.ofNullable(extendedModuleAnnotation)
+                .map(a -> ResourceFactory.createResource(
+                        KBSS_MODULE.uri + a.label().replaceAll(" ", "-").toLowerCase()) //todo can be added to the annotation
+                ).ifPresent(r -> baseRdfModel.add(root, RDFS.subClassOf, r));
+
         baseRdfModel.add(root, RDF.type, SM.Module);
         baseRdfModel.add(root, RDFS.comment, moduleAnnotation.comment());
         baseRdfModel.add(root, RDFS.label, moduleAnnotation.label());
-        final var statements = baseRdfModel.listStatements(null, RDF.type, SM.Module);
-        while (statements.hasNext()) {
-            final var statement = statements.next();
-            final var subject = statement.getSubject();
-            for (var annotation : constraintAnnotations) {
-                final var modelConstraint = ResourceFactory.createResource();
-                baseRdfModel.add(modelConstraint, RDF.type, SPL.Argument);
-                baseRdfModel.add(modelConstraint, SPL.predicate, annotation.urlPrefix() + annotation.name());
-                baseRdfModel.add(modelConstraint, RDFS.comment, "Automatically generated field: " + annotation.name());
-                baseRdfModel.add(subject, SPIN.constraint, modelConstraint);
-
-                getLog().debug("Added model constraint based on annotation: " +
-                        "(name = " + annotation.name() + ", urlPrefix = " + annotation.urlPrefix() + ")");
-            }
+        for (var annotation : constraintAnnotations) {
+            final var modelConstraint = ResourceFactory.createResource();
+            baseRdfModel.add(modelConstraint, RDF.type, SPL.Argument);
+            baseRdfModel.add(modelConstraint, SPL.predicate, ResourceFactory.createResource(annotation.urlPrefix() + annotation.name()));
+            baseRdfModel.add(modelConstraint, RDFS.label, annotation.name());
+            baseRdfModel.add(modelConstraint, RDFS.comment, annotation.comment());
+            baseRdfModel.add(root, SPIN.constraint, modelConstraint);
+
+            getLog().debug("Added model constraint based on annotation: " +
+                    "(name = " + annotation.name() + ", urlPrefix = " + annotation.urlPrefix() + ")");
         }
     }
 

s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/Ckan2RdfModule.java

@@ -5,6 +5,7 @@
 import cz.cvut.kbss.jopa.model.descriptors.EntityDescriptor;
 import cz.cvut.spipes.constants.KBSS_MODULE;
 import cz.cvut.spipes.engine.ExecutionContext;
+import cz.cvut.spipes.modules.annotations.SPipesModule;
 import eu.trentorise.opendata.jackan.CkanClient;
 import eu.trentorise.opendata.jackan.exceptions.CkanException;
 import eu.trentorise.opendata.jackan.model.CkanCatalog;
@@ -26,27 +27,21 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@SPipesModule(label = "ckan2rdf-v1", comment = "Convert ckan to rdf.")
 public class Ckan2RdfModule extends AnnotatedAbstractModule {
 
     public static final String TYPE_URI = KBSS_MODULE.uri + "ckan2rdf-v1";
     public static final String NS_DDO = "http://onto.fel.cvut.cz/ontologies/dataset-descriptor/";
     private static final Logger LOG = LoggerFactory.getLogger(Ckan2RdfModule.class);
-    /**
-     * URL of the CKAN server.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-ckan-url")
+
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-ckan-url", comment = "URL of the CKAN server.")
     private String propCkanApiUrl;
 
-    /**
-     * URL of the RDF4J repository.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-rdf4j-repository-url")
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-rdf4j-repository-url", comment = "URL of the RDF4J repository.")
     private String propRdf4jRepositoryUrl;
 
-    /**
-     * URL of the RDF4J repository.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-max-datasets")
+    // TODO - revise comment
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-max-datasets", comment = "Limits the number of processed datasets.")
     private Integer maxDatasets = Integer.MAX_VALUE;
 
     private Resource createInstance(final String classIri, final Model m) {

s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/SparqlEndpointDatasetExplorerModule.java

@@ -5,6 +5,8 @@
 import cz.cvut.spipes.engine.ExecutionContextFactory;
 import java.time.Instant;
 import java.util.Calendar;
+
+import cz.cvut.spipes.modules.annotations.SPipesModule;
 import org.apache.commons.io.IOUtils;
 import org.apache.jena.query.ParameterizedSparqlString;
 import org.apache.jena.query.Query;
@@ -18,29 +20,21 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@SPipesModule(label = "sparqlEndpointDatasetExplorer-v1", comment = "TODO")
 public class SparqlEndpointDatasetExplorerModule extends AnnotatedAbstractModule {
 
     public static final String TYPE_URI = KBSS_MODULE.uri + "sparqlEndpointDatasetExplorer-v1";
     private static final Logger LOG =
         LoggerFactory.getLogger(SparqlEndpointDatasetExplorerModule.class);
     private final String nsHttp = "http://onto.fel.cvut.cz/ontologies/http/";
 
-    /**
-     * URL of the SPARQL endpoint.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-sparql-endpoint-url")
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-sparql-endpoint-url", comment = "URL of the SPARQL endpoint.")
     private String propSparqlEndpointUrl;
 
-    /**
-     * Connection Timeout.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-connection-timeout")
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-connection-timeout", comment = "Connection Timeout in ms. Default 3000.")
     private long propConnectionTimeout = 3000;
 
-    /**
-     * Query Timeout.
-     */
-    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-query-timeout")
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "p-query-timeout", comment = "Query Timeout. Default 60000")
     private long propQueryTimeout = 60000;
 
     @Override

s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/DatasetDiscoveryModule.java

@@ -10,6 +10,8 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.TimeZone;
+
+import cz.cvut.spipes.modules.annotations.SPipesModule;
 import org.apache.jena.query.QueryExecution;
 import org.apache.jena.query.QueryExecutionFactory;
 import org.apache.jena.query.QueryFactory;
@@ -24,16 +26,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@SPipesModule(label = "dataset discovery v1", comment =
+        "Discovers dataset based on keyword userInput in repository linked.opendata.cz-federated-descriptor-faceted-search " +
+        "hosted at http://onto.fel.cvut.cz/rdf4j-server.")
 public class DatasetDiscoveryModule extends AbstractModule {
 
     private static final Logger LOG = LoggerFactory.getLogger(DatasetDiscoveryModule.class);
 
     private static final String TYPE_URI = KBSS_MODULE.uri + "dataset-discovery-v1";
 
-    /**
-     * URL of the Sesame server.
-     */
     private static final Property P_USER_INPUT = getParameter("prp-user-input");
+    @Parameter(urlPrefix = TYPE_URI + "/", name = "prp-user-input", comment = "Keywords query. Keywords are separated by space.")
     private String userInput;
 
     private static Property getParameter(final String name) {

s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/GetDatasetDescriptorsModule.java

@@ -2,13 +2,8 @@
 
 import cz.cvut.spipes.constants.KBSS_MODULE;
 import cz.cvut.spipes.engine.ExecutionContext;
-import java.io.IOException;
-import java.util.Collections;
-import org.apache.jena.query.Query;
-import org.apache.jena.query.QueryExecution;
-import org.apache.jena.query.QueryExecutionFactory;
-import org.apache.jena.query.QueryFactory;
-import org.apache.jena.query.Syntax;
+import cz.cvut.spipes.modules.annotations.SPipesModule;
+import org.apache.jena.query.*;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.Property;
 import org.apache.jena.rdf.model.ResourceFactory;
@@ -19,21 +14,28 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+
+@SPipesModule(label = "get dataset descriptors v1", comment = "Retrieve dataset descriptor for dataset with dataset-iri in endpoint-url.")
 public class GetDatasetDescriptorsModule extends AbstractModule {
 
     private static final Logger LOG = LoggerFactory.getLogger(GetDatasetDescriptorsModule.class);
 
     private static final String TYPE_URI = KBSS_MODULE.uri + "get-dataset-descriptors-v1";
+    private static final String PARAM_URI = TYPE_URI + "/";
 
     /**
      * URL of the Sesame server.
      */
     private static final Property P_DATASET_IRI = getParameter("p-dataset-iri");
+    private static final Property P_ENDPOINT_URL = getParameter("endpoint-url");
+
+    @Parameter(urlPrefix = PARAM_URI, name = "p-dataset-iri", comment = "IRI of the dataset.")// TODO - revise comment
     private String prpDatasetIri;
 
-    /**
-     * URL of the SPARQL endpoint.
-     */
+    @Parameter(urlPrefix = PARAM_URI, name = "endpoint-url", comment = "URL of the SPARQL endpoint. Default value is 'http://onto.fel.cvut.cz/rdf4j-server/repositories/descriptors-metadata'")
     private String endpointUrl = "http://onto.fel.cvut.cz/rdf4j-server/repositories/descriptors-metadata";
 
     private static Property getParameter(final String name) {
@@ -95,5 +97,6 @@ public String getTypeURI() {
     @Override
     public void loadConfiguration() {
         prpDatasetIri = this.getStringPropertyValue(P_DATASET_IRI);
+        endpointUrl = Optional.ofNullable(this.getStringPropertyValue(P_ENDPOINT_URL)).orElse(endpointUrl);
     }
 }

s-pipes-modules/module-eccairs/src/main/java/cz/cvut/spipes/modules/ImportE5XModule.java

@@ -15,6 +15,7 @@
 import cz.cvut.spipes.engine.ExecutionContext;
 import cz.cvut.spipes.engine.ExecutionContextFactory;
 import cz.cvut.spipes.exception.ResourceNotFoundException;
+import cz.cvut.spipes.modules.annotations.SPipesModule;
 import cz.cvut.spipes.modules.eccairs.EccairsAccessFactory;
 import cz.cvut.spipes.modules.eccairs.JopaPersistenceUtils;
 import cz.cvut.spipes.modules.eccairs.SesameDataDao;
@@ -34,10 +35,16 @@
 import java.net.URI;
 import java.util.Arrays;
 
+@SPipesModule(label = "import e5x", comment = "Convert e5x xml files to rdf.")
 public class ImportE5XModule extends AbstractModule {
 
     private static final Logger LOG = LoggerFactory.getLogger(ImportE5XModule.class);
 
+    // TODO - this parameter id defined with IRI <http://onto.fel.cvut.cz/ontologies/lib/module-param/has-resource-uri> in  s-pipes-modules\module.sms.ttl
+    // TODO - we should be able to annotate directly "StreamResource e5xResource" instead
+    @Parameter(name = "has-resource-uri", comment = "Uri of a resource referencing content of an e5x file.")
+    private String e5xResourceUriStr;
+
     StreamResource e5xResource;
 
     private boolean computeEccairsToAviationSafetyOntologyMapping = true;
@@ -145,7 +152,7 @@ public String getTypeURI() {
 
     @Override
     public void loadConfiguration() {
-        String e5xResourceUriStr = getEffectiveValue(KBSS_MODULE.has_resource_uri).asLiteral().toString();
+        e5xResourceUriStr = getEffectiveValue(KBSS_MODULE.has_resource_uri).asLiteral().toString();
         e5xResource = getResourceByUri(e5xResourceUriStr);
     }
 

s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructFormMetadataModule.java

@@ -4,6 +4,7 @@
 import cz.cvut.spipes.constants.KBSS_MODULE;
 import cz.cvut.spipes.constants.SML;
 import cz.cvut.spipes.engine.ExecutionContext;
+import cz.cvut.spipes.modules.annotations.SPipesModule;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.jena.rdf.model.*;
 import org.apache.jena.vocabulary.RDF;
@@ -18,9 +19,7 @@
 import static cz.cvut.spipes.form.JenaFormUtils.getAnswerOrigin;
 import static cz.cvut.spipes.form.JenaFormUtils.getQuestionOrigin;
 
-/**
- * Compute form:has-origin-path and form:has-origin-path-id properties.
- */
+@SPipesModule(label = "construct form metadata", comment = "Compute form:has-origin-path and form:has-origin-path-id properties.")
 public class ConstructFormMetadataModule extends AnnotatedAbstractModule {
 
     private static final Logger LOG = LoggerFactory.getLogger(ConstructFormMetadataModule.class);
@@ -29,7 +28,7 @@ public class ConstructFormMetadataModule extends AnnotatedAbstractModule {
     private static final String PATH_SEPARATOR = ",";
     private static final String INSTANCE_TYPE_SEPARATOR = "|";
 
-    @Parameter(urlPrefix = SML.uri, name = "replace")
+    @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default value is false.")
     private boolean isReplace = false;
 
     private enum Origin {