diff --git a/s-pipes-core/src/main/java/cz/cvut/spipes/modules/Parameter.java b/s-pipes-core/src/main/java/cz/cvut/spipes/modules/Parameter.java index b35a4305..2ac63263 100644 --- a/s-pipes-core/src/main/java/cz/cvut/spipes/modules/Parameter.java +++ b/s-pipes-core/src/main/java/cz/cvut/spipes/modules/Parameter.java @@ -11,4 +11,5 @@ public @interface Parameter { String urlPrefix() default KBSS_MODULE.uri; String name(); + String comment(); } diff --git a/s-pipes-modules-utils/s-pipes-module-creator-maven-plugin/src/main/java/RdfAnnotationProcessorMojo.java b/s-pipes-modules-utils/s-pipes-module-creator-maven-plugin/src/main/java/RdfAnnotationProcessorMojo.java index f46e8afe..2a8bd491 100644 --- a/s-pipes-modules-utils/s-pipes-module-creator-maven-plugin/src/main/java/RdfAnnotationProcessorMojo.java +++ b/s-pipes-modules-utils/s-pipes-module-creator-maven-plugin/src/main/java/RdfAnnotationProcessorMojo.java @@ -139,9 +139,12 @@ private void generateRdfForAllModules() throws MalformedURLException, ClassNotFo for (Class moduleClass : moduleClasses) { getLog().info("Creating RDF for module '" + moduleClass.getCanonicalName() + "'"); var moduleAnnotation = readModuleAnnotationFromClass(moduleClass); + var extendedModuleAnnotation = readExtendedModuleAnnotationFromClass(moduleClass); var constraints = readConstraintsFromClass(moduleClass); - writeConstraintsToModel(model, constraints, moduleAnnotation); + writeConstraintsToModel(model, constraints, moduleAnnotation, extendedModuleAnnotation); } + Optional.ofNullable(readManuallyManagedModuleDescriptionOntology(submodule)) + .ifPresent(model::add); getLog().info("--------------------------------------"); } @@ -154,14 +157,27 @@ private void generateRdfForAllModules() throws MalformedURLException, ClassNotFo getLog().info("======================================"); } + private SPipesModule readExtendedModuleAnnotationFromClass(Class moduleClass) { + + SPipesModule ret = null; + Class cls = moduleClass.getSuperclass(); + while(ret == null && cls != null) { + ret = readModuleAnnotationFromClass(cls); + cls = cls.getSuperclass(); + } + + return ret; + } + private void generateRdfForModule() throws MojoExecutionException { try { Set> moduleClasses = readAllModuleClasses(this.project); var model = readModelFromDefaultFile(); for (Class moduleClass : moduleClasses) { var moduleAnnotation = readModuleAnnotationFromClass(moduleClass); + var extendedModuleAnnotation = readExtendedModuleAnnotationFromClass(moduleClass); var constraints = readConstraintsFromClass(moduleClass); - writeConstraintsToModel(model, constraints, moduleAnnotation); + writeConstraintsToModel(model, constraints, moduleAnnotation, extendedModuleAnnotation); } var ontologyPath = modulePackageName.replaceAll("[.]", "/") + "/" + ontologyFilename; @@ -203,10 +219,45 @@ private Set> readAllModuleClasses(MavenProject project) throws Malforme return moduleClasses; } + /** + * Reads manually managed modules ontology from maven sub-project. + * The ontology is loaded from the resource folder of the sub-project and it is expected to start with the artifact + * id of the sub-project followed by the postfix ".tll". For example, assume the input parameter project + * artifact id "s-pipes-modules-text-analysis". This method will look for the ontology located at: + *
   $resource-dir$/s-pipes-modules-text-analysis.ttl
+ * + * Any ontology resources and their triples are removed from the model before returning. + * @param project + * @return + */ + private Model readManuallyManagedModuleDescriptionOntology(MavenProject project){ + String ontoName = project.getArtifactId() + ".ttl"; + Optional ontoUri = Optional.ofNullable(project).map(p -> + p.getResources().stream() + .map(r -> new File(r.getDirectory(), ontoName)) + .filter(File::exists) + .map(f -> f.getAbsoluteFile().toURI().toString()) + .findFirst().orElse(null) + ); + + if (!ontoUri.isPresent()) + return null; + + Model m = ModelFactory.createDefaultModel(); + m.read(ontoUri.get(), "TTL"); + m.listSubjectsWithProperty(RDF.type, OWL.Ontology).toList() + .forEach(o -> m.removeAll(o, null, null)); + + return m; + } + private URL[] getDependencyURLs(MavenProject project) throws MalformedURLException { Set ret = new HashSet<>(); ret.add(new File(project.getBuild().getOutputDirectory()).toURI().toURL()); - for(Dependency d : project.getDependencies()){ + + for(Dependency d : project.getDependencies().stream() + .filter(d -> !Artifact.SCOPE_TEST.equals(d.getScope())) + .collect(Collectors.toList())){ URL dURL = getURL(getLocalRepository(project), d); ret.add(dURL); } @@ -239,10 +290,21 @@ private SPipesModule readModuleAnnotationFromClass(Class classObject) { } private List readConstraintsFromClass(Class classObject) { - return Arrays.stream(classObject.getDeclaredFields()) - .filter(field -> field.isAnnotationPresent(PARAM_ANNOTATION)) - .map(field -> field.getAnnotation(PARAM_ANNOTATION)) - .collect(Collectors.toUnmodifiableList()); + List parameterConstraints = new ArrayList<>(); + + Class cls = classObject; + while(cls != null){ + if(cls != classObject && cls.isAnnotationPresent(MODULE_ANNOTATION)) { + cls = cls.getSuperclass(); + continue; + } + Arrays.stream(cls.getDeclaredFields()) + .filter(field -> field.isAnnotationPresent(PARAM_ANNOTATION)) + .map(field -> field.getAnnotation(PARAM_ANNOTATION)) + .forEach(parameterConstraints::add); + cls = cls.getSuperclass(); + } + return parameterConstraints; } //endregion @@ -269,25 +331,28 @@ private Model readModelFromDefaultFile() { private void writeConstraintsToModel(Model baseRdfModel, List constraintAnnotations, - SPipesModule moduleAnnotation) { + SPipesModule moduleAnnotation, + SPipesModule extendedModuleAnnotation) { final var root = ResourceFactory.createResource(KBSS_MODULE.uri + moduleAnnotation.label().replaceAll(" ", "-").toLowerCase()); //todo can be added to the annotation + // set extended uri + Optional.ofNullable(extendedModuleAnnotation) + .map(a -> ResourceFactory.createResource( + KBSS_MODULE.uri + a.label().replaceAll(" ", "-").toLowerCase()) //todo can be added to the annotation + ).ifPresent(r -> baseRdfModel.add(root, RDFS.subClassOf, r)); + baseRdfModel.add(root, RDF.type, SM.Module); baseRdfModel.add(root, RDFS.comment, moduleAnnotation.comment()); baseRdfModel.add(root, RDFS.label, moduleAnnotation.label()); - final var statements = baseRdfModel.listStatements(null, RDF.type, SM.Module); - while (statements.hasNext()) { - final var statement = statements.next(); - final var subject = statement.getSubject(); - for (var annotation : constraintAnnotations) { - final var modelConstraint = ResourceFactory.createResource(); - baseRdfModel.add(modelConstraint, RDF.type, SPL.Argument); - baseRdfModel.add(modelConstraint, SPL.predicate, annotation.urlPrefix() + annotation.name()); - baseRdfModel.add(modelConstraint, RDFS.comment, "Automatically generated field: " + annotation.name()); - baseRdfModel.add(subject, SPIN.constraint, modelConstraint); - - getLog().debug("Added model constraint based on annotation: " + - "(name = " + annotation.name() + ", urlPrefix = " + annotation.urlPrefix() + ")"); - } + for (var annotation : constraintAnnotations) { + final var modelConstraint = ResourceFactory.createResource(); + baseRdfModel.add(modelConstraint, RDF.type, SPL.Argument); + baseRdfModel.add(modelConstraint, SPL.predicate, ResourceFactory.createResource(annotation.urlPrefix() + annotation.name())); + baseRdfModel.add(modelConstraint, RDFS.label, annotation.name()); + baseRdfModel.add(modelConstraint, RDFS.comment, annotation.comment()); + baseRdfModel.add(root, SPIN.constraint, modelConstraint); + + getLog().debug("Added model constraint based on annotation: " + + "(name = " + annotation.name() + ", urlPrefix = " + annotation.urlPrefix() + ")"); } } diff --git a/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/Ckan2RdfModule.java b/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/Ckan2RdfModule.java index 3ab01f2c..4217489f 100644 --- a/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/Ckan2RdfModule.java +++ b/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/Ckan2RdfModule.java @@ -5,6 +5,7 @@ import cz.cvut.kbss.jopa.model.descriptors.EntityDescriptor; import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; +import cz.cvut.spipes.modules.annotations.SPipesModule; import eu.trentorise.opendata.jackan.CkanClient; import eu.trentorise.opendata.jackan.exceptions.CkanException; import eu.trentorise.opendata.jackan.model.CkanCatalog; @@ -26,27 +27,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SPipesModule(label = "ckan2rdf-v1", comment = "Convert ckan to rdf.") public class Ckan2RdfModule extends AnnotatedAbstractModule { public static final String TYPE_URI = KBSS_MODULE.uri + "ckan2rdf-v1"; public static final String NS_DDO = "http://onto.fel.cvut.cz/ontologies/dataset-descriptor/"; private static final Logger LOG = LoggerFactory.getLogger(Ckan2RdfModule.class); - /** - * URL of the CKAN server. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-ckan-url") + + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-ckan-url", comment = "URL of the CKAN server.") private String propCkanApiUrl; - /** - * URL of the RDF4J repository. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-rdf4j-repository-url") + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-rdf4j-repository-url", comment = "URL of the RDF4J repository.") private String propRdf4jRepositoryUrl; - /** - * URL of the RDF4J repository. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-max-datasets") + // TODO - revise comment + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-max-datasets", comment = "Limits the number of processed datasets.") private Integer maxDatasets = Integer.MAX_VALUE; private Resource createInstance(final String classIri, final Model m) { diff --git a/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/SparqlEndpointDatasetExplorerModule.java b/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/SparqlEndpointDatasetExplorerModule.java index ca135b4c..6d8e07a4 100644 --- a/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/SparqlEndpointDatasetExplorerModule.java +++ b/s-pipes-modules/module-ckan2rdf/src/main/java/cz/cvut/spipes/modules/SparqlEndpointDatasetExplorerModule.java @@ -5,6 +5,8 @@ import cz.cvut.spipes.engine.ExecutionContextFactory; import java.time.Instant; import java.util.Calendar; + +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.commons.io.IOUtils; import org.apache.jena.query.ParameterizedSparqlString; import org.apache.jena.query.Query; @@ -18,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SPipesModule(label = "sparqlEndpointDatasetExplorer-v1", comment = "TODO") public class SparqlEndpointDatasetExplorerModule extends AnnotatedAbstractModule { public static final String TYPE_URI = KBSS_MODULE.uri + "sparqlEndpointDatasetExplorer-v1"; @@ -25,22 +28,13 @@ public class SparqlEndpointDatasetExplorerModule extends AnnotatedAbstractModule LoggerFactory.getLogger(SparqlEndpointDatasetExplorerModule.class); private final String nsHttp = "http://onto.fel.cvut.cz/ontologies/http/"; - /** - * URL of the SPARQL endpoint. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-sparql-endpoint-url") + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-sparql-endpoint-url", comment = "URL of the SPARQL endpoint.") private String propSparqlEndpointUrl; - /** - * Connection Timeout. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-connection-timeout") + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-connection-timeout", comment = "Connection Timeout in ms. Default 3000.") private long propConnectionTimeout = 3000; - /** - * Query Timeout. - */ - @Parameter(urlPrefix = TYPE_URI + "/", name = "p-query-timeout") + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-query-timeout", comment = "Query Timeout. Default 60000") private long propQueryTimeout = 60000; @Override diff --git a/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/DatasetDiscoveryModule.java b/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/DatasetDiscoveryModule.java index e52d37be..bb21ca9f 100644 --- a/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/DatasetDiscoveryModule.java +++ b/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/DatasetDiscoveryModule.java @@ -10,6 +10,8 @@ import java.util.ArrayList; import java.util.List; import java.util.TimeZone; + +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; import org.apache.jena.query.QueryFactory; @@ -24,16 +26,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SPipesModule(label = "dataset discovery v1", comment = + "Discovers dataset based on keyword userInput in repository linked.opendata.cz-federated-descriptor-faceted-search " + + "hosted at http://onto.fel.cvut.cz/rdf4j-server.") public class DatasetDiscoveryModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(DatasetDiscoveryModule.class); private static final String TYPE_URI = KBSS_MODULE.uri + "dataset-discovery-v1"; - /** - * URL of the Sesame server. - */ private static final Property P_USER_INPUT = getParameter("prp-user-input"); + @Parameter(urlPrefix = TYPE_URI + "/", name = "prp-user-input", comment = "Keywords query. Keywords are separated by space.") private String userInput; private static Property getParameter(final String name) { diff --git a/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/GetDatasetDescriptorsModule.java b/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/GetDatasetDescriptorsModule.java index ee12f0ef..691df6a2 100644 --- a/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/GetDatasetDescriptorsModule.java +++ b/s-pipes-modules/module-dataset-discovery/src/main/java/cz/cvut/spipes/modules/GetDatasetDescriptorsModule.java @@ -2,13 +2,8 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; -import java.io.IOException; -import java.util.Collections; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryExecution; -import org.apache.jena.query.QueryExecutionFactory; -import org.apache.jena.query.QueryFactory; -import org.apache.jena.query.Syntax; +import cz.cvut.spipes.modules.annotations.SPipesModule; +import org.apache.jena.query.*; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResourceFactory; @@ -19,21 +14,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +@SPipesModule(label = "get dataset descriptors v1", comment = "Retrieve dataset descriptor for dataset with dataset-iri in endpoint-url.") public class GetDatasetDescriptorsModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(GetDatasetDescriptorsModule.class); private static final String TYPE_URI = KBSS_MODULE.uri + "get-dataset-descriptors-v1"; + private static final String PARAM_URI = TYPE_URI + "/"; /** * URL of the Sesame server. */ private static final Property P_DATASET_IRI = getParameter("p-dataset-iri"); + private static final Property P_ENDPOINT_URL = getParameter("endpoint-url"); + + @Parameter(urlPrefix = PARAM_URI, name = "p-dataset-iri", comment = "IRI of the dataset.")// TODO - revise comment private String prpDatasetIri; - /** - * URL of the SPARQL endpoint. - */ + @Parameter(urlPrefix = PARAM_URI, name = "endpoint-url", comment = "URL of the SPARQL endpoint. Default value is 'http://onto.fel.cvut.cz/rdf4j-server/repositories/descriptors-metadata'") private String endpointUrl = "http://onto.fel.cvut.cz/rdf4j-server/repositories/descriptors-metadata"; private static Property getParameter(final String name) { @@ -95,5 +97,6 @@ public String getTypeURI() { @Override public void loadConfiguration() { prpDatasetIri = this.getStringPropertyValue(P_DATASET_IRI); + endpointUrl = Optional.ofNullable(this.getStringPropertyValue(P_ENDPOINT_URL)).orElse(endpointUrl); } } diff --git a/s-pipes-modules/module-eccairs/src/main/java/cz/cvut/spipes/modules/ImportE5XModule.java b/s-pipes-modules/module-eccairs/src/main/java/cz/cvut/spipes/modules/ImportE5XModule.java index 39b72dcc..6b2fa24b 100644 --- a/s-pipes-modules/module-eccairs/src/main/java/cz/cvut/spipes/modules/ImportE5XModule.java +++ b/s-pipes-modules/module-eccairs/src/main/java/cz/cvut/spipes/modules/ImportE5XModule.java @@ -15,6 +15,7 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; import cz.cvut.spipes.exception.ResourceNotFoundException; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.modules.eccairs.EccairsAccessFactory; import cz.cvut.spipes.modules.eccairs.JopaPersistenceUtils; import cz.cvut.spipes.modules.eccairs.SesameDataDao; @@ -34,10 +35,16 @@ import java.net.URI; import java.util.Arrays; +@SPipesModule(label = "import e5x", comment = "Convert e5x xml files to rdf.") public class ImportE5XModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ImportE5XModule.class); + // TODO - this parameter id defined with IRI in s-pipes-modules\module.sms.ttl + // TODO - we should be able to annotate directly "StreamResource e5xResource" instead + @Parameter(name = "has-resource-uri", comment = "Uri of a resource referencing content of an e5x file.") + private String e5xResourceUriStr; + StreamResource e5xResource; private boolean computeEccairsToAviationSafetyOntologyMapping = true; @@ -145,7 +152,7 @@ public String getTypeURI() { @Override public void loadConfiguration() { - String e5xResourceUriStr = getEffectiveValue(KBSS_MODULE.has_resource_uri).asLiteral().toString(); + e5xResourceUriStr = getEffectiveValue(KBSS_MODULE.has_resource_uri).asLiteral().toString(); e5xResource = getResourceByUri(e5xResourceUriStr); } diff --git a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructFormMetadataModule.java b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructFormMetadataModule.java index 61050f32..b8d4ce41 100644 --- a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructFormMetadataModule.java +++ b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructFormMetadataModule.java @@ -4,6 +4,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.commons.codec.digest.DigestUtils; import org.apache.jena.rdf.model.*; import org.apache.jena.vocabulary.RDF; @@ -18,9 +19,7 @@ import static cz.cvut.spipes.form.JenaFormUtils.getAnswerOrigin; import static cz.cvut.spipes.form.JenaFormUtils.getQuestionOrigin; -/** - * Compute form:has-origin-path and form:has-origin-path-id properties. - */ +@SPipesModule(label = "construct form metadata", comment = "Compute form:has-origin-path and form:has-origin-path-id properties.") public class ConstructFormMetadataModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ConstructFormMetadataModule.class); @@ -29,7 +28,7 @@ public class ConstructFormMetadataModule extends AnnotatedAbstractModule { private static final String PATH_SEPARATOR = ","; private static final String INSTANCE_TYPE_SEPARATOR = "|"; - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default value is false.") private boolean isReplace = false; private enum Origin { diff --git a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructTextualViewModule.java b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructTextualViewModule.java index beac6e3e..bceeb6fd 100644 --- a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructTextualViewModule.java +++ b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/ConstructTextualViewModule.java @@ -12,6 +12,7 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.form.JenaFormUtils; import cz.cvut.spipes.form.JopaPersistenceUtils; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; @@ -21,32 +22,37 @@ import java.util.List; import java.util.function.Predicate; -/** - * For input Q&A models constructs textual view of specified questions. Each textual view represent the question - * and its sub-questions recursively. - */ +@SPipesModule(label = "construct textual view", comment = + "For input Q&A models constructs textual view of specified questions. Each textual" + + " view represent the question and its sub-questions recursively." +) public class ConstructTextualViewModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ConstructTextualViewModule.class); private static final String TYPE_URI = KBSS_MODULE.uri + "construct-textual-view"; - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default value is false.") private boolean isReplace = false; - @Parameter(urlPrefix = FORM_MODULE.uri, name = "serialize-unanswered-questions") + @Parameter(urlPrefix = FORM_MODULE.uri, name = "serialize-unanswered-questions", + comment = "If true (default), outputs questions not having answers") private boolean isSerializeUnansweredQuestions = true; - @Parameter(urlPrefix = FORM_MODULE.uri, name = "serialize-answers") + @Parameter(urlPrefix = FORM_MODULE.uri, name = "serialize-answers", + comment = "If true (default), outputs answers not only questions") private boolean isSerializeAnswers = true; - @Parameter(urlPrefix = FORM_MODULE.uri, name = "process-non-root-questions") + @Parameter(urlPrefix = FORM_MODULE.uri, name = "process-non-root-questions", + comment = "If true (default), process all questions, otherwise process only root questions") private boolean isProcessNonRootQuestions = true; - @Parameter(urlPrefix = FORM_MODULE.uri, name = "indentation-string") + @Parameter(urlPrefix = FORM_MODULE.uri, name = "indentation-string", + comment = "Indentation string indents subquestion from questions. By default ' ' is used.") private String indentationString = " "; - @Parameter(urlPrefix = FORM_MODULE.uri, name = "language") + @Parameter(urlPrefix = FORM_MODULE.uri, name = "language", + comment = "Language to be used to retrieve labels of questions. By default 'en' is used.") private String language = "en"; @Override diff --git a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/FetchPossibleValuesModule.java b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/FetchPossibleValuesModule.java index ade11676..e85f2e6f 100644 --- a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/FetchPossibleValuesModule.java +++ b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/FetchPossibleValuesModule.java @@ -6,6 +6,7 @@ import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.form.JenaFormUtils; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; @@ -15,16 +16,16 @@ import java.util.HashMap; import java.util.Map; -/** - * Inputs are forms using Q&A model. Possible values of questions are added to questions that does not have - * any value attached and contains possible value query. - */ +@SPipesModule(label = "fetch possible values", comment = + "Fetches possible values for answers of questions. Inputs are forms using Q&A model. Possible values of " + + "questions are added to questions that does not have any value attached and contains possible value query." +) public class FetchPossibleValuesModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(FetchPossibleValuesModule.class); private static final String TYPE_URI = KBSS_MODULE.uri + "fetch-possible-values"; - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag, default is false.") //TODO - revise comment private boolean isReplace = false; diff --git a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/MergeFormMetadataModule.java b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/MergeFormMetadataModule.java index 73acbe14..18bc50e0 100644 --- a/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/MergeFormMetadataModule.java +++ b/s-pipes-modules/module-form/src/main/java/cz/cvut/spipes/modules/MergeFormMetadataModule.java @@ -6,6 +6,7 @@ import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.form.JenaFormUtils; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.commons.codec.digest.DigestUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -15,11 +16,11 @@ import java.util.Random; -/** - * Inputs are sample form and Q&A model. Questions from both models are remapped to new IRIs based on - * question origin combined with executionId. New question instances are created using questionInstanceTemplate - * which defaults to "doc:question-{_questionOriginHash}-{_executionId}". - */ +@SPipesModule(label = "merge form metadata", comment = + "Merges form metadata. Inputs are sample form and Q&A model. Questions from both models are remapped to new" + + "IRIs based on question origin combined with executionId. New question instances are created using" + + "questionInstanceTemplate which defaults to \"doc:question-{_questionOriginHash}-{_executionId}\"." +) public class MergeFormMetadataModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(MergeFormMetadataModule.class); @@ -30,13 +31,15 @@ public class MergeFormMetadataModule extends AnnotatedAbstractModule { private static final String QUESTION_ORIGIN_HASH_VAR = "{_questionOriginHash}"; private static final String EXECUTION_ID_VAR = "{_executionId}"; - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default value is false.") // TODO - revise comment private boolean isReplace = false; - @Parameter(name = "execution-id") + @Parameter(name = "execution-id", comment = "Execution id that will be used to construct question IRIs") private String executionId = DigestUtils.md5Hex(Long.toString(RANDOM.nextLong())); - @Parameter(name = "question-instance-template") + // TODO - revise comment + @Parameter(name = "question-instance-template", comment = "URL Template to create URL for question instances. " + + "Default is 'http://onto.fel.cvut.cz/ontologies/documentation/question-{_questionOriginHash}-{_executionId}'") private String questionInstanceTemplate = SFormsVocabularyJena.s_c_question.toString() + "-" diff --git a/s-pipes-modules/module-identity/src/main/java/cz/cvut/spipes/modules/IdentityModule.java b/s-pipes-modules/module-identity/src/main/java/cz/cvut/spipes/modules/IdentityModule.java index 9c0b152d..0179de55 100644 --- a/s-pipes-modules/module-identity/src/main/java/cz/cvut/spipes/modules/IdentityModule.java +++ b/s-pipes-modules/module-identity/src/main/java/cz/cvut/spipes/modules/IdentityModule.java @@ -2,9 +2,11 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SPipesModule(label = "identity", comment = "Implements a no-op.") public class IdentityModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(IdentityModule.class); diff --git a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModule.java b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModule.java index f69afa6a..dd7b1e16 100644 --- a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModule.java +++ b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModule.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.sutime.AnnforModel; import cz.cvut.spipes.sutime.DescriptorModel; import edu.stanford.nlp.ling.CoreAnnotations; @@ -27,13 +28,17 @@ import java.nio.file.Paths; +@SPipesModule(label = "temporal v0.1", comment = "Annotate temporal expressions in literals in input model.") public class SUTimeModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(SUTimeModule.class); public static final String TYPE_URI = KBSS_MODULE.getURI() + "temporal-v0.1"; + @Parameter(urlPrefix = DescriptorModel.prefix, name = "has-rule-file", comment = "Rule file, multivalued.") // TODO - revise comment private List ruleFilePaths = new LinkedList<>(); + + @Parameter(urlPrefix = DescriptorModel.prefix, name = "has-document-date", comment = "Document date format.") // TODO - revise comment private String documentDate; // TODO support other formats ? diff --git a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModuleNew.java b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModuleNew.java index 272aa9f1..9d319b5e 100644 --- a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModuleNew.java +++ b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/modules/SUTimeModuleNew.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.sutime.AnnforModel; import cz.cvut.spipes.sutime.DescriptorModel; import cz.cvut.spipes.util.JenaUtils; @@ -33,26 +34,38 @@ import java.util.*; +@SPipesModule(label = "temporal-v1", comment = "Module annotates input triples using NLP analysis of time using library SUTime.") public class SUTimeModuleNew extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(SUTimeModuleNew.class); - private static final String TYPE_URI = KBSS_MODULE.getURI() + "temporal-v1"; + private static final String TYPE_URI = KBSS_MODULE.uri + "temporal-v1"; private static final String TYPE_PREFIX = TYPE_URI + "/"; private static final int DEFAULT_PAGE_SIZE = 10000; private static final String LIMIT_OFFSET_CLAUSE_MARKER_NAME = "LIMIT_OFFSET"; private static final Property P_PAGE_SIZE = ResourceFactory.createProperty(TYPE_PREFIX + "page-size"); + + @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size", comment = "Page size. Default value is 10000.") private Integer pageSize = DEFAULT_PAGE_SIZE; + + @Parameter(urlPrefix = SML.uri, name = "constructQuery", + comment = "List of construct queries. The module annotates the lexical form of objects of the output statements of these queries.")// TODO - revise comment private List constructQueries; + //sml:replace + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default value is false." ) private boolean isReplace; //kbss:parseText - /** - * Whether the query should be taken from sp:text property instead of from SPIN serialization - */ + @Parameter(urlPrefix = KBSS_MODULE.uri, name = "is-parse-text", + comment = "Whether the query should be taken from sp:text property instead of from SPIN serialization," + + " default is true.") private boolean parseText; + + @Parameter(urlPrefix = DescriptorModel.prefix, name = "has-rule-file", comment = "Rule file, multivalued.")// TODO - review comment private List ruleFilePaths = new LinkedList<>(); + + @Parameter(urlPrefix = DescriptorModel.prefix, name = "has-document-date", comment = "Document date format.")// TODO - review comment private String documentDate; // TODO support other formats ? private AnnotationPipeline pipeline; diff --git a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/sutime/DescriptorModel.java b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/sutime/DescriptorModel.java index 95edfe89..239141e6 100644 --- a/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/sutime/DescriptorModel.java +++ b/s-pipes-modules/module-nlp/src/main/java/cz/cvut/spipes/sutime/DescriptorModel.java @@ -9,7 +9,7 @@ public class DescriptorModel { /** * The namespace of the vocabulary as a string */ - private static final String prefix = "http://onto.fel.cvut.cz/ontologies/dataset-descriptor/temporal-v1/"; + public static final String prefix = "http://onto.fel.cvut.cz/ontologies/dataset-descriptor/temporal-v1/"; protected static final Resource resource(String local ) { return ResourceFactory.createResource( prefix + local ); } diff --git a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jCreateRepositoryModule.java b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jCreateRepositoryModule.java index 3de9ea58..8faf53fe 100644 --- a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jCreateRepositoryModule.java +++ b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jCreateRepositoryModule.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.exceptions.RepositoryAlreadyExistsException; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResourceFactory; import org.eclipse.rdf4j.repository.config.RepositoryConfig; @@ -15,30 +16,25 @@ import java.util.Objects; -/** - * Module creates native store rdf4j repository on the given server with the given name - */ +@SPipesModule(label = "rdf4j create repository", comment = "Module creates native store rdf4j repository on the given server with the given name.") public class Rdf4jCreateRepositoryModule extends AbstractModule { - private static final Logger LOG = LoggerFactory.getLogger(Rdf4jUpdateModule.class.getName()); - private static final String TYPE_URI = KBSS_MODULE.getURI() + "rdf4j-create-repository"; - private static final String PROPERTY_PREFIX_URI = KBSS_MODULE.getURI() + "rdf4j"; + private static final Logger LOG = LoggerFactory.getLogger(Rdf4jCreateRepositoryModule.class.getName()); + private static final String TYPE_URI = KBSS_MODULE.uri + "rdf4j-create-repository"; + private static final String PROPERTY_PREFIX_URI = KBSS_MODULE.uri + "rdf4j"; - /** - * URL of the Rdf4j server - */ static final Property P_RDF4J_SERVER_URL = getParameter("p-rdf4j-server-url"); + + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-server-url", comment = "URL of the Rdf4j server") private String rdf4jServerURL; - /** - * Rdf4j repository ID - */ static final Property P_RDF4J_REPOSITORY_NAME = getParameter("p-rdf4j-repository-name"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-repository-name", comment = "Rdf4j repository ID") private String rdf4jRepositoryName; - /** - * Don't try to create new repository if it already exists (Default value is false) - */ static final Property P_RDF4J_IGNORE_IF_EXISTS = getParameter("p-rdf4j-ignore-if-exists"); + + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-ignore-if-exists", + comment = "Don't try to create new repository if it already exists (Default value is false)") private boolean rdf4jIgnoreIfExists; private RepositoryManager repositoryManager; diff --git a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jDeployModule.java b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jDeployModule.java index df5c51b8..6dbce062 100644 --- a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jDeployModule.java +++ b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jDeployModule.java @@ -4,6 +4,7 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; import cz.cvut.spipes.exception.ModuleConfigurationInconsistentException; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.util.CoreConfigProperies; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResourceFactory; @@ -30,41 +31,35 @@ import java.io.StringWriter; import java.util.Optional; -/** - * Module deploys content of input execution context - * into default context of repository (if p-rdf4j-context-iri is not specified) - * or concrete context (if p-rdf4j-context-iri is specified). - */ +@SPipesModule(label = "deploy", comment = + "Module deploys content of input execution context into default context of repository (if p-rdf4j-context-iri " + + "is not specified) or concrete context (if p-rdf4j-context-iri is specified)." +) public class Rdf4jDeployModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(Rdf4jDeployModule.class); - private static String TYPE_URI = KBSS_MODULE.getURI()+"deploy"; - private static String PROPERTY_PREFIX_URI = KBSS_MODULE.getURI()+"rdf4j"; + private final static String TYPE_URI = KBSS_MODULE.uri + "deploy"; + private final static String PROPERTY_PREFIX_URI = KBSS_MODULE.uri + "rdf4j"; private static Property getParameter(final String name) { return ResourceFactory.createProperty(PROPERTY_PREFIX_URI + "/" + name); } - /** - * URL of the Rdf4j server - */ static final Property P_RDF4J_SERVER_URL = getParameter("p-rdf4j-server-url"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-server-url", comment = "URL of the Rdf4j server") private String rdf4jServerURL; - /** - * Rdf4j repository ID - */ static final Property P_RDF4J_REPOSITORY_NAME = getParameter("p-rdf4j-repository-name"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-repository-name", comment = "Rdf4j repository ID") private String rdf4jRepositoryName; - /** - * IRI of the context that should be used for deployment - */ static final Property P_RDF4J_CONTEXT_IRI = getParameter("p-rdf4j-context-iri"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-context-iri", comment = "IRI of the context that should be used for deployment.") private String rdf4jContextIRI; static final Property P_RDF4J_REPOSITORY_USERNAME = getParameter("p-rdf4j-secured-username-variable"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-secured-username-variable", comment = "User name if the repository requires authentication.") private String rdf4jSecuredUsernameVariable; private RepositoryManager repositoryManager; private Repository repository; @@ -74,12 +69,13 @@ public void setRepositoryManager(RepositoryManager repositoryManager) { } static final Property P_RDF4J_REPOSITORY_PASSWORD = getParameter("p-rdf4j-secured-password-variable"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-secured-password-variable", comment = "Password if the repository requires authentication.") private String rdf4jSecuredPasswordVariable; - /** - * Whether data should be replaced (true) / appended (false) into the specified context or repository. - * Default is false. - */ + static final Property P_IS_REPLACE_CONTEXT_IRI = getParameter("p-is-replace"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-is-replace", comment = + "Whether data should be replaced (true) / appended (false) into the specified context or repository.\n" + + "Default is false.") private boolean isReplaceContext; public String getRdf4jServerURL() { diff --git a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jUpdateModule.java b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jUpdateModule.java index 034eb606..bc039f6f 100644 --- a/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jUpdateModule.java +++ b/s-pipes-modules/module-rdf4j/src/main/java/cz/cvut/spipes/modules/Rdf4jUpdateModule.java @@ -5,6 +5,7 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.exception.ModuleConfigurationInconsistentException; import cz.cvut.spipes.exceptions.RepositoryAccessException; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.util.QueryUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Property; @@ -26,28 +27,33 @@ import java.util.List; import java.util.stream.Collectors; +@SPipesModule(label = "rdf4j update", comment = "Updates sparql endpoint configured in rdf4jServerURL using specified updateQueries.") public class Rdf4jUpdateModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(Rdf4jUpdateModule.class.getName()); - private static final String TYPE_URI = KBSS_MODULE.getURI() + "rdf4j-update"; - private static final String PROPERTY_PREFIX_URI = KBSS_MODULE.getURI() + "rdf4j"; + private static final String TYPE_URI = KBSS_MODULE.uri + "rdf4j-update"; + private static final String PROPERTY_PREFIX_URI = KBSS_MODULE.uri + "rdf4j"; - /** - * URL of the Rdf4j server - */ static final Property P_RDF4J_SERVER_URL = getParameter("p-rdf4j-server-url"); + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-server-url", comment = "URL of the Rdf4j server") private String rdf4jServerURL; - /** - * Rdf4j repository ID - */ static final Property P_RDF4J_REPOSITORY_NAME = getParameter("p-rdf4j-repository-name"); + + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-rdf4j-repository-name", comment = "Rdf4j repository ID") private String rdf4jRepositoryName; + + @Parameter(urlPrefix = SML.uri, name = "updateQuery", comment = "A list of SPARQL Update queries. Queries strings are read from sp:text.")// TODO - revise comment private List updateQueries; static final Property P_RDF4J_STOP_ITERATION_ON_STABLE_TRIPLE_COUNT = getParameter("p-stop-iteration-on-stable-triple-count"); + + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "p-stop-iteration-on-stable-triple-count", + comment = "Stops iteration on if triple count did not change. Default is false.") // TODO - revise comment private boolean onlyIfTripleCountChanges; + @Parameter(urlPrefix = PROPERTY_PREFIX_URI + "/", name = "has-max-iteration-count", + comment = "Limits the number of iterations to the specified value. Default value is 1.") // TODO - revise comment private int iterationCount; private Repository updateRepository; diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructAbstractModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructAbstractModule.java index bd0ecc62..d78227fd 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructAbstractModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructAbstractModule.java @@ -23,32 +23,39 @@ public abstract class ApplyConstructAbstractModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ApplyConstructAbstractModule.class); - + private static final String TYPE_URI = KBSS_MODULE.uri + "abstract-apply-construct"; + private static final String PROPERTY_PREFIX_URI = KBSS_MODULE.uri + ""; //sml:constructQuery + // TODO - this parameter is reused in ApplyConstructWithChunkedValuesAndScrollableCursorModule. There the comment should be extended by a note, i.e. "The construct queries with markers #${VALUES} and #${LIMIT_OFFSET}." + @Parameter(urlPrefix = SML.uri, name = "constructQuery", comment = "List of construct queries the output of which is returned by the module.") //TODO - revise comment protected List constructQueries; //sml:replace + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag, default is false.") protected boolean isReplace; //kbss:parseText - /** - * Whether the query should be taken from sp:text property instead of from SPIN serialization - */ + @Parameter(urlPrefix = KBSS_MODULE.uri, name = "is-parse-text", + comment = "Whether the query should be taken from sp:text property instead of from SPIN serialization," + + " default is true." + ) protected boolean parseText; //kbss:iterationCount - /** - * Maximal number of iterations of the whole rule set. 0 means 0 iterations. The actual number of iterations can be smaller, - * if no new inferences are generated any more. - *

- * iterationCount = 1: - * - the whole rule set is executed only once. - * iterationCount > 1: - * - the whole rule set is executed at most "iterationCount" times. - * - in each iteration, queries are evaluated on the model merged from the default model and the result of previous iteration - *

- * Within each iteration, all queries are evaluated on the same model. - */ + @Parameter(name = "has-max-iteration-count", + comment = + "Maximal number of iterations of the whole rule set. 0 means 0 iterations. The actual number of iterations can be smaller,\n" + + "if no new inferences are generated any more.\n" + + "

\n" + + "iterationCount = 1:\n" + + "- the whole rule set is executed only once.\n" + + "iterationCount > 1:\n" + + "- the whole rule set is executed at most \"iterationCount\" times.\n" + + "- in each iteration, queries are evaluated on the model merged from the default model and the result of previous iteration\n" + + "

\n" + + "Within each iteration, all queries are evaluated on the same model." + + ) protected int iterationCount = -1; @@ -175,7 +182,7 @@ public void loadConfiguration() { //TODO default value must be taken from template definition isReplace = this.getPropertyValue(SML.replace, false); - parseText = this.getPropertyValue(KBSS_MODULE.is_parse_text, false); + parseText = this.getPropertyValue(KBSS_MODULE.is_parse_text, true); iterationCount = this.getPropertyValue(KBSS_MODULE.has_max_iteration_count, -1); } diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructV2Module.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructV2Module.java index 694301a7..5d34f56c 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructV2Module.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructV2Module.java @@ -1,17 +1,19 @@ package cz.cvut.spipes.modules; import cz.cvut.spipes.constants.KBSS_MODULE; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * TODO Order of queries is not enforced. */ +@SPipesModule(label = "apply construct v2", comment = "Generates triples from input model using specified constructQueries.") public class ApplyConstructV2Module extends ApplyConstructAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ApplyConstructV2Module.class); - private static final String TYPE_URI = KBSS_MODULE.uri + "apply-contruct-v2"; + private static final String TYPE_URI = KBSS_MODULE.uri + "apply-construct-v2"; @Override public String getTypeURI() { diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesAndScrollableCursorModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesAndScrollableCursorModule.java index 874b32d9..7caa8604 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesAndScrollableCursorModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesAndScrollableCursorModule.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.VariablesBinding; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.recursion.ChunkedValuesProvider; import cz.cvut.spipes.recursion.CombinedQueryTemplateRecursionProvider; import cz.cvut.spipes.recursion.QueryTemplateRecursionProvider; @@ -26,6 +27,7 @@ * TODO issue with redundant call {@link ScrollableCursorProvider} * TODO supports only one CONSTRUCT query */ +@SPipesModule(label = "apply construct with chunked values and scrollable cursor", comment = "Apply construct with chunked values and scrollable cursor") public class ApplyConstructWithChunkedValuesAndScrollableCursorModule extends ApplyConstructAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ApplyConstructWithChunkedValuesAndScrollableCursorModule.class); @@ -37,9 +39,14 @@ public class ApplyConstructWithChunkedValuesAndScrollableCursorModule extends Ap private static final Property P_CHUNK_SIZE = ResourceFactory.createProperty(TYPE_PREFIX + "chunk-size"); private static final Property P_PAGE_SIZE = ResourceFactory.createProperty(TYPE_PREFIX + "page-size"); + @Parameter(urlPrefix = TYPE_PREFIX, name = "chunk-size", comment = "Chunk size. Default is 10.") private Integer chunkSize = DEFAULT_CHUNK_SIZE; + @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size", comment = "Page size for the scrollable cursor. Default is 10000.") private Integer pageSize = DEFAULT_PAGE_SIZE; + + @Parameter(urlPrefix = SML.uri, name = "selectQuery", + comment = "The select query that will be used to iterate over construct query templates.") private Select selectQuery; diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesModule.java index aa86e0c0..98ce7e82 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithChunkedValuesModule.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.VariablesBinding; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.util.QueryUtils; import java.util.Objects; import org.apache.jena.query.Query; @@ -22,17 +23,22 @@ /** * TODO Order of queries is not enforced. */ +@SPipesModule(label = "apply construct with chunked values", comment = "Apply construct with chunked values.") public class ApplyConstructWithChunkedValuesModule extends ApplyConstructAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ApplyConstructWithChunkedValuesModule.class); - private static final String TYPE_URI = KBSS_MODULE.uri + "apply-construct-with-chunked-values"; + private static final String TYPE_URI = KBSS_MODULE.uri + ""; private static final String TYPE_PREFIX = TYPE_URI + "/"; private static final int DEFAULT_CHUNK_SIZE = 10; private static final String VALUES_CLAUSE_MARKER_NAME = "VALUES"; private static final Property P_CHUNK_SIZE = ResourceFactory.createProperty(TYPE_PREFIX + "chunk-size"); + @Parameter(urlPrefix = TYPE_PREFIX, name = "chunk-size", comment = "Chunk size. Default is 10.") private Integer chunkSize = DEFAULT_CHUNK_SIZE; + + @Parameter(urlPrefix = SML.uri, name = "selectQuery" + , comment = "The select query that will be used to iterate over construct query templates.") private Select selectQuery; diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithScrollableCursorModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithScrollableCursorModule.java index 4988fea2..bf6754f5 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithScrollableCursorModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ApplyConstructWithScrollableCursorModule.java @@ -1,6 +1,7 @@ package cz.cvut.spipes.modules; import cz.cvut.spipes.constants.KBSS_MODULE; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.util.QueryUtils; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Property; @@ -11,6 +12,12 @@ /** * TODO Order of queries is not enforced. */ +@SPipesModule(label = "apply construct with scrollable cursor", + comment = "Runs one or more construct queries (bound to sml:constructQuery) on the input triples. Queries are " + + "executed multiple times with scrollable cursor that is injected through query marker #${LIMIT_OFFSET}. " + + "The marker is replaced each time with appropriate by sparql constructs 'LIMIT ?limit' and 'OFFSET ?offset'. " + + "Within each construct query The output RDF will consist of the constructed triples and (unless sml:replace is true) " + + "the input triples.") public class ApplyConstructWithScrollableCursorModule extends ApplyConstructAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ApplyConstructWithScrollableCursorModule.class); @@ -21,6 +28,7 @@ public class ApplyConstructWithScrollableCursorModule extends ApplyConstructAbst private static final String LIMIT_OFFSET_CLAUSE_MARKER_NAME = "LIMIT_OFFSET"; private static final Property P_PAGE_SIZE = ResourceFactory.createProperty(TYPE_PREFIX + "page-size"); + @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size", comment = "Page size. Default value is 10000.") private Integer pageSize = DEFAULT_PAGE_SIZE; @Override diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/DownloadGraphModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/DownloadGraphModule.java index 61fca7e9..587473b5 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/DownloadGraphModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/DownloadGraphModule.java @@ -10,6 +10,8 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; + +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ResourceFactory; import org.apache.jena.riot.Lang; @@ -17,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SPipesModule(label = "sparql endpoint download graph", comment = "Downloads named graph namedGraphId from sparql endpoint endpointUrl.") public class DownloadGraphModule extends AnnotatedAbstractModule { private static final String TYPE_URI = KBSS_MODULE.uri + "sparql-endpoint-download-graph"; @@ -24,16 +27,16 @@ public class DownloadGraphModule extends AnnotatedAbstractModule { private static final int DEFAULT_PAGE_SIZE = 10000; private static final Logger LOG = LoggerFactory.getLogger(DownloadGraphModule.class); - @Parameter(urlPrefix = TYPE_PREFIX, name = "named-graph-id") + @Parameter(urlPrefix = TYPE_PREFIX, name = "named-graph-id", comment = "Named graph id") private String namedGraphId; - @Parameter(urlPrefix = TYPE_PREFIX, name = "endpoint-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "endpoint-url", comment = "Endpoint url") private String endpointUrl; - @Parameter(urlPrefix = TYPE_PREFIX, name = "output-resource-variable") + @Parameter(urlPrefix = TYPE_PREFIX, name = "output-resource-variable", comment = "Output resource variable") private String outputResourceVariable; - @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size") + @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size", comment = "Page size. Default value is 10000.") private Integer pageSize = DEFAULT_PAGE_SIZE; protected long numberOfDownloadedTriples; diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ExternalSchemExModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ExternalSchemExModule.java index 02698527..46fb4741 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ExternalSchemExModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ExternalSchemExModule.java @@ -4,6 +4,7 @@ import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.util.ExecUtils; import java.io.IOException; import java.io.InputStream; @@ -13,6 +14,7 @@ import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.util.FileUtils; +@SPipesModule(label = "external schemex", comment = "Compute schemex using external script for a specified sourceFilePath.") public class ExternalSchemExModule extends AbstractModule { private static final String MODULE_ID = "external-schemex"; @@ -20,6 +22,7 @@ public class ExternalSchemExModule extends AbstractModule { private static final String TYPE_PREFIX = TYPE_URI + "/"; private static final String SCHEMEX_PROGRAM = "schemex"; //sml:sourceFilePath + @Parameter(urlPrefix = SML.uri, name = "sourceFilePath", comment = "Source file in nt format.") private Path sourceFilePath; @Override diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ImproveSPOWithMarginalsModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ImproveSPOWithMarginalsModule.java index 2fe7c794..626c6457 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ImproveSPOWithMarginalsModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/ImproveSPOWithMarginalsModule.java @@ -5,6 +5,7 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; import cz.cvut.spipes.engine.VariablesBinding; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.tdb.TDBTempFactory; import cz.cvut.spipes.util.JenaUtils; import cz.cvut.spipes.util.QueryUtils; @@ -40,6 +41,13 @@ /** * TODO Order of queries is not enforced. */ +@SPipesModule(label = "improve spo with marginals", comment = "Constructs improved spo-summary descriptor with knowledge " + + "of provided marginals of weakly described resources. This module expects as an input graph computed spo-summary " + + "patterns (or possibly whole spo-summary descriptor) compliant with data provided in ?data-service-url. Within the " + + "input graph it identifies 'breakable patterns', i.e. spo-summary patterns that can be improved with knowledge of " + + "marginals computed in ?marginals-defs-file-url. The output of the module is a spo-summary descriptor that contains " + + "original spo-summary patterns whenever possible and new spo-summary patterns that were created with additional " + + "knowledge of marginals.") public class ImproveSPOWithMarginalsModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ImproveSPOWithMarginalsModule.class); @@ -48,13 +56,14 @@ public class ImproveSPOWithMarginalsModule extends AnnotatedAbstractModule { private static final String TYPE_URI = KBSS_MODULE.uri + MODULE_ID; private static final String TYPE_PREFIX = TYPE_URI + "/"; private static Map marginalDefsModelCache = new HashMap<>(); - //@Parameter(urlPrefix = TYPE_PREFIX, name = "marginal-constraint") + + @Parameter(urlPrefix = TYPE_PREFIX, name = "marginal-constraint", comment = "Marginal constraint") private String marginalConstraint; - //@Parameter(urlPrefix = TYPE_PREFIX, name = "marginals-defs-file-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "marginals-defs-file-url", comment = "Marginal definitions file url") private String marginalsDefsFileUrl; - //@Parameter(urlPrefix = TYPE_PREFIX, name = "marginals-file-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "marginals-file-url", comment = "Marginals file url") // TODO - review comment private String marginalsFileUrl; - //@Parameter(urlPrefix = TYPE_PREFIX, name = "data-service-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "data-service-url", comment = "Data service url") private String dataServiceUrl; private static final String VAR_EXECUTION_ID = "executionId"; diff --git a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/RetrieveGraphModule.java b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/RetrieveGraphModule.java index 8b91bcb2..a244e22f 100644 --- a/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/RetrieveGraphModule.java +++ b/s-pipes-modules/module-sparql-endpoint/src/main/java/cz/cvut/spipes/modules/RetrieveGraphModule.java @@ -4,11 +4,16 @@ import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; import cz.cvut.spipes.impl.GraphChunkedDownload; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +// TODO - lable = SpEP retrieve graph +@SPipesModule(label = "sparql endpoint retrieve graph", comment = "Retrieves graph from sparql endpoint specified by " + + "?endpointUrl and optionaly ?namedGraphId. If ?namedGraphId is not specified it retreaves the default graph.") public class RetrieveGraphModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(RetrieveGraphModule.class); @@ -16,13 +21,13 @@ public class RetrieveGraphModule extends AnnotatedAbstractModule { private static final String TYPE_PREFIX = TYPE_URI + "/"; private static final int DEFAULT_PAGE_SIZE = 10000; - @Parameter(urlPrefix = TYPE_PREFIX, name = "named-graph-id") + @Parameter(urlPrefix = TYPE_PREFIX, name = "named-graph-id", comment = "Named graph id") private String namedGraphId; - @Parameter(urlPrefix = TYPE_PREFIX, name = "endpoint-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "endpoint-url", comment = "Endpoint url") private String endpointUrl; - @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size") + @Parameter(urlPrefix = TYPE_PREFIX, name = "page-size", comment = "Page size. Default is 10000.") private Integer pageSize = DEFAULT_PAGE_SIZE; public String getNamedGraphId() { diff --git a/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/RDF2CSVModule.java b/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/RDF2CSVModule.java index a16ddc2c..44aa27f6 100644 --- a/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/RDF2CSVModule.java +++ b/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/RDF2CSVModule.java @@ -5,6 +5,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.*; import org.apache.jena.riot.RDFDataMgr; import org.apache.jena.riot.RDFFormat; @@ -22,13 +23,11 @@ import java.util.Optional; import java.util.stream.Collectors; -/** - * Module for converting RDF (representing table) to CSV - *

- * The module is responsible for converting the input RDF data into a CSV format and saving the output to a file. - * The table is constructed from column and row resources defined in TableSchema and saves it as a new CSV file. - *

- */ +@SPipesModule(label = "RDF2CSV", comment = "Module for converting RDF (representing table) to CSV\n" + + "

\n" + + "The module is responsible for converting the input RDF data into a CSV format and saving the output to a file.\n" + + "The table is constructed from column and row resources defined in TableSchema and saves it as a new CSV file.\n" + + "

") public class RDF2CSVModule extends AnnotatedAbstractModule { public static final String TYPE_URI = KBSS_MODULE.uri + "RDF2CSV"; @@ -36,8 +35,8 @@ public class RDF2CSVModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(RDF2CSVModule.class); - /** The parameter representing where the output file will be stored */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "file-output-path") + @Parameter(urlPrefix = TYPE_PREFIX, name = "file-output-path", + comment = "The parameter representing where the output file will be stored.") private String fileOutputPath; @Override diff --git a/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/TabularModule.java b/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/TabularModule.java index 3bc7dd9f..6cb1a4f4 100644 --- a/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/TabularModule.java +++ b/s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/TabularModule.java @@ -101,6 +101,7 @@ public class TabularModule extends AbstractModule { public static final String TYPE_URI = KBSS_MODULE.uri + "tabular"; + public static final String PARAM_URL_PREFIX = TYPE_URI + "/"; private static final Logger LOG = LoggerFactory.getLogger(TabularModule.class); private final Property P_DELIMITER = getSpecificParameter("delimiter"); private final Property P_QUOTE_CHARACTER = getSpecificParameter("quote-character"); @@ -113,21 +114,27 @@ public class TabularModule extends AbstractModule { private final Property P_PROCESS_SPECIFIC_SHEET_IN_XLS_FILE = getSpecificParameter("process-specific-sheet-in-xls-file"); //sml:replace + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag. Default is false.")// TODO - revise comment private boolean isReplace; + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "source-resource-uri", comment = "Resource URI of the source table.") // TODO - revise comment //:source-resource-uri private StreamResource sourceResource; //:delimiter + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "delimiter", comment = "Column delimiter. Default value is comma ','.") private int delimiter; //:quote-character + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "quote-character", comment = "Quote character. Default is '\"' if delimiter is ',', '\\0' otherwize.") private char quoteCharacter; //:data-prefix + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "data-prefix", comment = "Data prefix")// TODO - improve comment private String dataPrefix; //:skip-header + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "skip-header", comment = "Skip header. Default is false.") private boolean skipHeader; //:process-specific-sheet-in-xls-file @@ -137,6 +144,8 @@ public class TabularModule extends AbstractModule { private int processSpecificSheetInXLSFile; //:output-mode + // TODO - revise comment + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "output-mode", comment = "Output mode. Default is standard-mode('http://onto.fel.cvut.cz/ontologies/lib/module/tabular/standard-mode)") private Mode outputMode; //:source-resource-format @@ -152,6 +161,7 @@ public class TabularModule extends AbstractModule { private ResourceFormat sourceResourceFormat = ResourceFormat.PLAIN; //:accept-invalid-quoting + @Parameter(urlPrefix = PARAM_URL_PREFIX, name = "accept-invalid-quoting", comment = "Accept invalid quoting. Default is false.") private boolean acceptInvalidQuoting; /** diff --git a/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/ModuleTarql.java b/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/ModuleTarql.java index 7aa93301..ad6e8bf6 100644 --- a/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/ModuleTarql.java +++ b/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/ModuleTarql.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; +import cz.cvut.spipes.modules.annotations.SPipesModule; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Property; @@ -16,32 +17,27 @@ import java.nio.file.Paths; @Deprecated //TODO merge with TarqlModule functionality +@SPipesModule(label = "tarql-XXX-2", comment = "Module to convert CSV file to RDF and query it using SPRQL query. The module wraps org.deri.tarql.tarql. This module is depracated.") public class ModuleTarql extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ModuleTarql.class); - private static final String TYPE_URI = KBSS_MODULE.getURI() + "tarql" + "-XXX-2"; + private static final String TYPE_URI = KBSS_MODULE.uri + "tarql" + "-XXX-2"; private static Property getParameter(final String name) { - return ResourceFactory.createProperty(TYPE_URI + name); + return ResourceFactory.createProperty(TYPE_URI + "/" + name); } - /** - * File with the TARQL script - */ static final Property P_TARQL_STRING = getParameter("p-tarql-string"); + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-tarql-string", comment = "File with the TARQL script." ) private String tarqlString; - /** - * Ontology IRI - */ static final Property P_ONTOLOGY_IRI = getParameter("p-ontology-iri"); + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-ontology-iri", comment = "Ontology IRI") private String ontologyIRI; - /** - * Input File - */ static final Property P_INPUT_FILE = getParameter("p-input-file"); + @Parameter(urlPrefix = TYPE_URI + "/", name = "p-input-file", comment = "Input File") private String inputFile; // /** diff --git a/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/TarqlModule.java b/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/TarqlModule.java index 3ad28935..a5881869 100644 --- a/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/TarqlModule.java +++ b/s-pipes-modules/module-tarql/src/main/java/cz/cvut/spipes/modules/TarqlModule.java @@ -5,6 +5,7 @@ import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; import cz.cvut.spipes.engine.ExecutionContextFactory; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.registry.StreamResource; import cz.cvut.spipes.registry.StreamResourceRegistry; import org.apache.jena.ext.com.google.common.io.Files; @@ -27,6 +28,8 @@ import java.util.stream.Collectors; // TODO merge with ModuleTarql functionality +@SPipesModule(label = "tarql", comment = "\"Runs one or more TARQL Construct queries on the input triples. The output RDF " + + "will consist of the constructed triples and (unless sml:replace is true) the input triples.\"") public class TarqlModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(TarqlModule.class); @@ -35,14 +38,20 @@ public class TarqlModule extends AbstractModule { private static final String TARQL_PROGRAM = AppConstants.BIN_DIR + "/tarql"; //sml:constructQuery + @Parameter(urlPrefix = SML.uri, name = "constructQuery", comment = "The TARQL Construct queries that deliver the triples that shall be added.") private List constructQueries; + // TODO not used field private String tableFilePath; //sml:replace + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "If set to true, the output triples will only contain the " + + "constructed triples. If no values or false are specified, the output will be the union of the input triples " + + "and the constructed triples.") private boolean isReplace; //sml:sourceFilePath + @Parameter(urlPrefix = SML.uri, name = "sourceFilePath", comment = "Source CSV file.") private String sourceFilePath; public TarqlModule() { diff --git a/s-pipes-modules/module-text-analysis/pom.xml b/s-pipes-modules/module-text-analysis/pom.xml index fd9339c3..e7a7f4b5 100644 --- a/s-pipes-modules/module-text-analysis/pom.xml +++ b/s-pipes-modules/module-text-analysis/pom.xml @@ -72,27 +72,4 @@ compile - - - - - cz.cvut.kbss - s-pipes-module-creator-maven-plugin - ${project.parent.version} - - - - process-annotations - - - RDF_FOR_MODULE - cz.cvut.spipes.modules - ExtractTermOccurrencesModule - module-text-analysis-generated.ttl - - - - - - diff --git a/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/ExtractTermOccurrencesModule.java b/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/ExtractTermOccurrencesModule.java index d4d94c55..ce7a3e10 100644 --- a/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/ExtractTermOccurrencesModule.java +++ b/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/ExtractTermOccurrencesModule.java @@ -3,6 +3,7 @@ import cz.cvut.spipes.constants.KBSS_MODULE; import cz.cvut.spipes.constants.SML; import cz.cvut.spipes.engine.ExecutionContext; +import cz.cvut.spipes.modules.annotations.SPipesModule; import cz.cvut.spipes.modules.constants.Termit; import cz.cvut.spipes.modules.textAnalysis.Extraction; import org.apache.commons.codec.digest.DigestUtils; @@ -21,55 +22,54 @@ import java.util.List; import java.util.Map; -/** - * Module extracts term occurrences from annotated literals of input RDF. - *

- * Annotated literals are RDF string literals that are annotated by RDFa - * using TermIt terminology to mark occurrences of terms within the text. - *

- * - * Example of usage: - *

- * Input: - *


- *  :x  a csvw:row
- *      :csat-wo-tc "4339272" ;
- *      :tc-reference "52-610-00-04" ;
- *      :wo-text "in the <span about=\"_:a877-16\" property=\"ddo:je-výskytem-termu\" resource=\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\" typeof=\"ddo:výskyt-termu\" score=\"0.25\">cockpit</span>" ;
- * 
- *

- * The expected output: - *

- *      <http://onto.fel.cvut.cz/ontologies/application/termit/pojem/výskyt-termu/instance2fc6112ce9a960c21918569bef6a4151> a       termit-pojem:výskyt-termu ;
- *         termit-pojem:je-přiřazením-termu <http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb> ;
- *         termit-pojem:má-cíl
- *               [ a       termit-pojem:cíl-výskytu ;
- *                 termit-pojem:má-selektor
- *                         [ a       termit-pojem:selektor-pozici-v-textu ;
- *                           termit-pojem:má-koncovou-pozici
- *                                   "14"^^<http://www.w3.org/2001/XMLSchema#int> ;
- *                           termit-pojem:má-startovní-pozici
- *                                    "7"^^<http://www.w3.org/2001/XMLSchema#int>
- *                         ] ;
- *                 termit-pojem:má-selektor
- *                         [ a       termit-pojem:selektor-text-quote> ;
- *                           termit-pojem:má-prefix-text-quote
- *                                    "in the " ;
- *                           termit-pojem:má-přesný-text-quote
- *                                    "cockpit" ;
- *                           termit-pojem:má-suffix-text-quote
- *                                     ""
- *                         ]
- *               ];
- *         termit-pojem:má-skóre
- *                 "0.25"^^<http://www.w3.org/2001/XMLSchema#float> ;
- *         termit-pojem:odkazuje-na-anotovaný-text
- *                 "in the <span about=\"_:a877-16\" property=\"ddo:je-výskytem-termu\" resource=\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\" typeof=\"ddo:výskyt-termu\" score=\"0.25\">cockpit</span>" ;
- *         termit-pojem:odkazuje-na-anotaci
- *                 "<span about=\"_:a877-16\" property=\"ddo:je-výskytem-termu\" resource=\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\" typeof=\"ddo:výskyt-termu\" score=\"0.25\">cockpit</span>" .
- * .
- * 
- */ +@SPipesModule(label = "extract term occurrences", comment = + "Module extracts term occurrences from annotated literals of input RDF.\n" + + "

\n" + + "Annotated literals are RDF string literals that are annotated by RDFa\n" + + "using TermIt terminology to mark occurrences of terms within the text.\n" + + "

\n" + + "Example of usage:\n" + + "

\n" + + "Input:\n" + + "

\n" +
+        " :x  a csvw:row\n" +
+        "     :csat-wo-tc \"4339272\" ;\n" +
+        "     :tc-reference \"52-610-00-04\" ;\n" +
+        "     :wo-text \"in the <span about=\\\"_:a877-16\\\" property=\\\"ddo:je-výskytem-termu\\\" resource=\\\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\\\" typeof=\\\"ddo:výskyt-termu\\\" score=\\\"0.25\\\">cockpit</span>\" ;\n" +
+        "
\n" + + "

\n" + + "The expected output:\n" + + "
\n" +
+        "     <http://onto.fel.cvut.cz/ontologies/application/termit/pojem/výskyt-termu/instance2fc6112ce9a960c21918569bef6a4151> a       termit-pojem:výskyt-termu ;\n" +
+        "        termit-pojem:je-přiřazením-termu <http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb> ;\n" +
+        "        termit-pojem:má-cíl\n" +
+        "              [ a       termit-pojem:cíl-výskytu ;\n" +
+        "                termit-pojem:má-selektor\n" +
+        "                        [ a       termit-pojem:selektor-pozici-v-textu ;\n" +
+        "                          termit-pojem:má-koncovou-pozici\n" +
+        "                                  \"14\"^^<http://www.w3.org/2001/XMLSchema#int> ;\n" +
+        "                          termit-pojem:má-startovní-pozici\n" +
+        "                                   \"7\"^^<http://www.w3.org/2001/XMLSchema#int>\n" +
+        "                        ] ;\n" +
+        "                termit-pojem:má-selektor\n" +
+        "                        [ a       termit-pojem:selektor-text-quote> ;\n" +
+        "                          termit-pojem:má-prefix-text-quote\n" +
+        "                                   \"in the \" ;\n" +
+        "                          termit-pojem:má-přesný-text-quote\n" +
+        "                                   \"cockpit\" ;\n" +
+        "                          termit-pojem:má-suffix-text-quote\n" +
+        "                                    \"\"\n" +
+        "                        ]\n" +
+        "              ];\n" +
+        "        termit-pojem:má-skóre\n" +
+        "                \"0.25\"^^<http://www.w3.org/2001/XMLSchema#float> ;\n" +
+        "        termit-pojem:odkazuje-na-anotovaný-text\n" +
+        "                \"in the <span about=\\\"_:a877-16\\\" property=\\\"ddo:je-výskytem-termu\\\" resource=\\\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\\\" typeof=\\\"ddo:výskyt-termu\\\" score=\\\"0.25\\\">cockpit</span>\" ;\n" +
+        "        termit-pojem:odkazuje-na-anotaci\n" +
+        "                \"<span about=\\\"_:a877-16\\\" property=\\\"ddo:je-výskytem-termu\\\" resource=\\\"http://onto.fel.cvut.cz/ontologies/slovnik/slovnik-komponent-a-zavad---novy/pojem/cvr-ulb\\\" typeof=\\\"ddo:výskyt-termu\\\" score=\\\"0.25\\\">cockpit</span>\" .\n" +
+        ".\n" +
+        "
" +) public class ExtractTermOccurrencesModule extends AnnotatedAbstractModule { private static final Logger LOG = LoggerFactory.getLogger(ExtractTermOccurrencesModule.class); @@ -77,8 +77,7 @@ public class ExtractTermOccurrencesModule extends AnnotatedAbstractModule { private static final String TYPE_URI = KBSS_MODULE.uri + "extract-term-occurrences"; private static final String TYPE_PREFIX = TYPE_URI + "/"; - /** Indicates whether the existing RDF should be overwritten. */ - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Indicates whether the existing RDF should be overwritten. Default false.") // TODO - revise comment private boolean isReplace; Extraction extraction = new Extraction(); diff --git a/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/TextAnalysisModule.java b/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/TextAnalysisModule.java index 4bd91603..534d93a4 100644 --- a/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/TextAnalysisModule.java +++ b/s-pipes-modules/module-text-analysis/src/main/java/cz/cvut/spipes/modules/TextAnalysisModule.java @@ -33,56 +33,52 @@ import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; import static org.apache.commons.lang.StringEscapeUtils.unescapeHtml; -/** - * Module for text analysis. - *

- * This class provides a module for text analysis. - * It uses an external web service to analyze text data and retrieve annotated text. - * It analyse the text using a SKOS vocabulary that is stored in RDF4J repository. - *

- */ -@SPipesModule(label = "Text analysis module", comment = "test comment") +@SPipesModule(label = "Text analysis module", comment = +"Module for text analysis.\n" + +"

\n" + +"This class provides a module for text analysis.\n" + +"It uses an external web service to analyze text data and retrieve annotated text.\n" + +"It analyse the text using a SKOS vocabulary that is stored in RDF4J repository.\n" + +"

" +) public class TextAnalysisModule extends AnnotatedAbstractModule{ private static final Logger LOG = LoggerFactory.getLogger(TextAnalysisModule.class); private static final String TYPE_URI = KBSS_MODULE.uri + "text-analysis"; private static final String TYPE_PREFIX = TYPE_URI + "/"; - /** The URL of the text analysis service to be used. */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "service-url") + @Parameter(urlPrefix = TYPE_PREFIX, name = "service-url", comment = "The URL of the text analysis service to be used.") private String serviceUrl; - /** The IRI of the vocabulary to be used for entity recognition. */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "vocabulary-iri") + @Parameter(urlPrefix = TYPE_PREFIX, name = "vocabulary-iri", comment = "The IRI of the vocabulary to be used for entity recognition.") private String vocabularyIri; - /** The IRI of the repository where the vocabulary is stored. */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "vocabulary-repository") + @Parameter(urlPrefix = TYPE_PREFIX, name = "vocabulary-repository", comment = "The IRI of the repository where the vocabulary is stored.") private String vocabularyRepository; - /** The language of the text to be analyzed. */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "language") + @Parameter(urlPrefix = TYPE_PREFIX, name = "language", comment = "The language of the text to be analyzed.") private String language; //sml:replace - @Parameter(urlPrefix = SML.uri, name = "replace") + @Parameter(urlPrefix = SML.uri, name = "replace", comment = "Replace context flag, default is false.") private boolean isReplace = false; - /** The number of literals to be processed per request to the web service. */ - @Parameter(urlPrefix = TYPE_PREFIX, name = "literals-per-request") + @Parameter(urlPrefix = TYPE_PREFIX, name = "literals-per-request", comment = "The number of literals to be processed per request to the web service.") private Integer literalsPerRequest; - /** The SPARQL query to be used for selecting literals from the repository. - *

- * Example: - *

{@code
-     * SELECT ?literal
-     * WHERE {
-     *    ?s ?p ?literal .
-     *    FILTER(isLiteral(?literal) && datatype(?literal) = xsd:string)
-     * }
-     * }
- */ + @Parameter(urlPrefix = SML.uri, name = "selectQuery", + comment = + "The SPARQL query to be used for selecting literals from the repository.\n" + + "

\n" + + "Example:\n" + + "

{@code\n" +
+                "SELECT ?literal\n" +
+                "WHERE {\n" +
+                "    ?s ?p ?literal .\n" +
+                "    FILTER(isLiteral(?literal) && datatype(?literal) = xsd:string)\n" +
+                "}\n" +
+                "}
" + ) private Select selectQuery; @Override