From da61e50a838b43ccaf6fe08f34e6c3f0c242a8cb Mon Sep 17 00:00:00 2001 From: Jens Pots Date: Fri, 24 May 2024 22:33:42 +0200 Subject: [PATCH] feat: new validator processor, fix: load class files --- build.gradle.kts | 3 + src/main/kotlin/exception/RunnerException.kt | 3 + src/main/kotlin/extensions/File.kt | 38 +++- src/main/kotlin/logging/Log.kt | 11 +- src/main/kotlin/std/RDFValidator.kt | 27 ++- src/main/resources/pipeline.ttl | 11 +- src/main/resources/std/file_reader.ttl | 3 +- src/main/resources/std/file_writer.ttl | 3 +- src/main/resources/std/http_fetch.ttl | 3 +- src/main/resources/std/rdf_validator.ttl | 3 +- src/test/kotlin/std/RDFValidatorTest.kt | 204 +++++++++++++++++++ src/test/resources/std | 1 + 12 files changed, 283 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/exception/RunnerException.kt create mode 100644 src/test/kotlin/std/RDFValidatorTest.kt create mode 120000 src/test/resources/std diff --git a/build.gradle.kts b/build.gradle.kts index 934afe1..9c07dc1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -21,6 +21,9 @@ repositories { mavenCentral() } dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0") + // Guava + implementation("com.google.guava:guava:33.2.0-jre") + // K2JVM Compiler. implementation("org.jetbrains.kotlin:kotlin-stdlib:1.9.22") implementation("org.jetbrains.kotlin:kotlin-compiler-embeddable:1.9.22") diff --git a/src/main/kotlin/exception/RunnerException.kt b/src/main/kotlin/exception/RunnerException.kt new file mode 100644 index 0000000..9390e8b --- /dev/null +++ b/src/main/kotlin/exception/RunnerException.kt @@ -0,0 +1,3 @@ +package technology.idlab.exception + +open class RunnerException : Exception() diff --git a/src/main/kotlin/extensions/File.kt b/src/main/kotlin/extensions/File.kt index 3268184..692d485 100644 --- a/src/main/kotlin/extensions/File.kt +++ b/src/main/kotlin/extensions/File.kt @@ -1,11 +1,16 @@ package technology.idlab.extensions +import com.google.common.reflect.ClassPath import java.io.File +import java.net.URLClassLoader import org.apache.jena.ontology.OntModelSpec import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.rdf.model.Resource import org.apache.jena.vocabulary.OWL +import org.jetbrains.kotlin.incremental.isClassFile +import org.jetbrains.kotlin.incremental.isJavaFile +import org.jetbrains.kotlin.incremental.isKotlinFile import technology.idlab.compiler.Compiler import technology.idlab.compiler.MemoryClassLoader import technology.idlab.logging.Log @@ -54,13 +59,32 @@ internal fun File.readModelRecursively(): Model { * Parse a file as a JVM processor by loading the class file from disk or compiling the source code. */ internal fun File.loadIntoJVM(): Class<*> { - val bytes = - when (extension) { - "kt" -> Compiler.compileKotlin(this) - "java" -> Compiler.compileJava(this) - "class" -> readBytes() - else -> Log.shared.fatal("Unsupported file extension: $extension") + if (this.isClassFile()) { + // Load the class file using a custom class loader. + val loader = URLClassLoader(arrayOf(toURI().toURL())) + val classPath = ClassPath.from(loader) + val classes = classPath.allClasses + + // Find the class which corresponds to this file. + for (clazz in classes) { + // TODO: this is a hack, we should use a better way to find the class. + if (clazz.name.endsWith(this.nameWithoutExtension)) { + return clazz.load() } + } + + Log.shared.fatal("Failed to load class ${this.nameWithoutExtension}") + } + + if (this.isKotlinFile(listOf("kt"))) { + val bytes = Compiler.compileKotlin(this) + return MemoryClassLoader().fromBytes(bytes, nameWithoutExtension) + } + + if (this.isJavaFile()) { + val bytes = Compiler.compileJava(this) + return MemoryClassLoader().fromBytes(bytes, nameWithoutExtension) + } - return MemoryClassLoader().fromBytes(bytes, nameWithoutExtension) + Log.shared.fatal("Unsupported file type: $extension") } diff --git a/src/main/kotlin/logging/Log.kt b/src/main/kotlin/logging/Log.kt index 0c9fd07..ce02576 100644 --- a/src/main/kotlin/logging/Log.kt +++ b/src/main/kotlin/logging/Log.kt @@ -4,7 +4,7 @@ import java.time.format.DateTimeFormatter import java.util.Date import java.util.TimeZone import kotlin.Exception -import kotlin.system.exitProcess +import technology.idlab.exception.RunnerException class Log private constructor() { init { @@ -64,20 +64,17 @@ class Log private constructor() { fun fatal(message: String): Nothing { print(message, "FATAL") - print(Throwable().stackTraceToString()) - exitProcess(1) + throw RunnerException() } fun fatal(exception: Exception): Nothing { print(exception.message.toString(), "FATAL") - print(Throwable().stackTraceToString()) - exitProcess(1) + throw RunnerException() } fun fatal(message: String, exception: Exception) { print("$message - ${exception.message}") - print(Throwable().stackTraceToString()) - exitProcess(1) + throw RunnerException() } fun debug(message: String) { diff --git a/src/main/kotlin/std/RDFValidator.kt b/src/main/kotlin/std/RDFValidator.kt index 7ba4dbf..b1780e6 100644 --- a/src/main/kotlin/std/RDFValidator.kt +++ b/src/main/kotlin/std/RDFValidator.kt @@ -6,6 +6,7 @@ import java.io.ByteArrayOutputStream import java.io.File import org.apache.jena.graph.Graph import org.apache.jena.rdf.model.ModelFactory +import org.apache.jena.riot.RiotException import org.apache.jena.shacl.ShaclValidator import technology.idlab.extensions.readModelRecursively import technology.idlab.logging.Log @@ -31,7 +32,14 @@ class RDFValidator(args: Map) : Processor(args) { init { val path = this.getArgument("shapes") val file = File(path) - val shapesModel = file.readModelRecursively() + + val shapesModel = + try { + file.readModelRecursively() + } catch (e: RiotException) { + Log.shared.fatal("Failed to read SHACL shapes from file://$path") + } + this.shapes = shapesModel.graph } @@ -46,17 +54,27 @@ class RDFValidator(args: Map) : Processor(args) { // Parse as a model. Log.shared.assert(model.isEmpty, "Model should be empty.") - model.read(res.value.toString()) + try { + model.read(res.value.inputStream(), null, "TURTLE") + } catch (e: RiotException) { + Log.shared.fatal("Failed to read incoming RDF data.") + } // Validate the model. val report = validator.validate(shapes, model.graph) - if (!report.conforms()) { + + if (report.conforms()) { + // Propagate to the output. + output.pushSync(res.value) + } else { + // Print the report if required. if (printReport.orElse(printReportDefault)) { val out = ByteArrayOutputStream() report.model.write(out, "TURTLE") Log.shared.info(out.toString()) } + // Check if we can continue after an error. if (errorIsFatal.orElse(errorIsFatalDefault)) { Log.shared.fatal("Validation error is fatal.") } @@ -64,9 +82,6 @@ class RDFValidator(args: Map) : Processor(args) { // Reset model for next invocation. model.removeAll(null, null, null) - - // Propagate to the output. - output.pushSync(res.value) } // Close the output. diff --git a/src/main/resources/pipeline.ttl b/src/main/resources/pipeline.ttl index fe9eb1c..e7001eb 100644 --- a/src/main/resources/pipeline.ttl +++ b/src/main/resources/pipeline.ttl @@ -1,11 +1,16 @@ @prefix jvm: . -@prefix fno: . -@prefix fnom: . @prefix xsd: . -@prefix : . @prefix sh: . @prefix rdf: . @prefix rdfs: . +@prefix owl: . + +# Include the Standard Processor Library +<> owl:imports + <./std/file_reader.ttl>, + <./std/file_writer.ttl>, + <./std/http_fetch.ttl>, + <./std/rdf_validator.ttl>. # Definition of a Processor. [] diff --git a/src/main/resources/std/file_reader.ttl b/src/main/resources/std/file_reader.ttl index bdc267d..f76755c 100644 --- a/src/main/resources/std/file_reader.ttl +++ b/src/main/resources/std/file_reader.ttl @@ -2,11 +2,12 @@ @prefix owl: . @prefix sh: . @prefix xsd: . +@prefix rdf: . <> owl:imports <../pipeline.ttl>. jvm:FileReader a jvm:Processor; - jvm:file <../../kotlin/std/FileReader.kt>; + jvm:file <../../../classes/kotlin/main/technology/idlab/std/FileReader.class>; jvm:language "Kotlin". [] a sh:NodeShape; diff --git a/src/main/resources/std/file_writer.ttl b/src/main/resources/std/file_writer.ttl index 2e11ed9..b131db1 100644 --- a/src/main/resources/std/file_writer.ttl +++ b/src/main/resources/std/file_writer.ttl @@ -2,11 +2,12 @@ @prefix owl: . @prefix sh: . @prefix xsd: . +@prefix rdf: . <> owl:imports <../pipeline.ttl>. jvm:FileWriter a jvm:Processor; - jvm:file <../../kotlin/std/FileWriter.kt>; + jvm:file <../../../classes/kotlin/main/technology/idlab/std/FileWriter.class>; jvm:language "Kotlin". [] a sh:NodeShape; diff --git a/src/main/resources/std/http_fetch.ttl b/src/main/resources/std/http_fetch.ttl index 9220d82..92f0bea 100644 --- a/src/main/resources/std/http_fetch.ttl +++ b/src/main/resources/std/http_fetch.ttl @@ -2,11 +2,12 @@ @prefix owl: . @prefix sh: . @prefix xsd: . +@prefix rdf: . <> owl:imports <../pipeline.ttl>. jvm:HttpFetch a jvm:Processor; - jvm:file <../../kotlin/std/HttpFetch.kt>; + jvm:file <../../../classes/kotlin/main/technology/idlab/std//HttpFetch.class>; jvm:language "Kotlin". [] a sh:NodeShape; diff --git a/src/main/resources/std/rdf_validator.ttl b/src/main/resources/std/rdf_validator.ttl index 3b773c3..a38d096 100644 --- a/src/main/resources/std/rdf_validator.ttl +++ b/src/main/resources/std/rdf_validator.ttl @@ -2,11 +2,12 @@ @prefix owl: . @prefix sh: . @prefix xsd: . +@prefix rdf: . <> owl:imports <../pipeline.ttl>. jvm:RDFValidator a jvm:Processor; - jvm:file <../../kotlin/std/RDFValidator.kt>; + jvm:file <../../../classes/kotlin/main/technology/idlab/std/RDFValidator.class>; jvm:language "Kotlin". [] a sh:NodeShape; diff --git a/src/test/kotlin/std/RDFValidatorTest.kt b/src/test/kotlin/std/RDFValidatorTest.kt new file mode 100644 index 0000000..6db9158 --- /dev/null +++ b/src/test/kotlin/std/RDFValidatorTest.kt @@ -0,0 +1,204 @@ +package std + +import bridge.DummyReader +import bridge.DummyWriter +import java.io.File +import java.util.Optional +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.text.toByteArray +import org.junit.jupiter.api.assertDoesNotThrow +import org.junit.jupiter.api.assertThrows +import technology.idlab.exception.RunnerException +import technology.idlab.runner.Pipeline +import technology.idlab.std.RDFValidator + +/** Ontology location. */ +private val ontology = + Thread.currentThread().contextClassLoader.getResource("pipeline.ttl")!!.let { File(it.file) } + +/** Textual representation of a valid SHACL file. */ +private const val validShape = + """ +@prefix ex: . +@prefix sh: . +@prefix rdf: . +@prefix xsd: . + +ex:PointShape + a sh:NodeShape; + sh:targetClass ex:Point; + sh:closed true; + sh:ignoredProperties (rdf:type); + sh:property [ + sh:path ex:x; + sh:message "Requires an integer X coordinate"; + sh:name "X-coordinate"; + sh:datatype xsd:int; + sh:minCount 1; + sh:maxCount 1; + ], [ + sh:path ex:y; + sh:message "Requires an integer Y coordinate"; + sh:name "Y-coordinate"; + sh:datatype xsd:int; + sh:minCount 1; + sh:maxCount 1; + ]. +""" + +/** Textual representation of an invalid SHACL file. */ +private val invalidShape = validShape.replace("xsd:int", "xkcd:int") + +/** Textual representation of valid RDF of a Point shape. */ +private const val validInput = + """ +@prefix ex: . +@prefix xsd: . + +ex:ValidPoint + a ex:Point; + ex:x "1"^^xsd:int; + ex:y "2"^^xsd:int. +""" + +/** File with invalid Point shape. */ +val invalidInput = validInput.replace("xsd:int", "xsd:string") + +/** A file containing a valid definition of SHACL shapes. */ +private val shapeFile = File.createTempFile("shapes", "ttl").apply { writeText(validShape) } + +/** A file containing an invalid definition of SHACL shapes. */ +private val invalidShapeFile = + File.createTempFile("invalid_shapes", "ttl").apply { writeText(invalidShape) } + +/** Simple pipeline containing the processor. */ +private val pipeline = + """ +@prefix jvm: . +@prefix xsd: . +@prefix owl: . + +<> owl:imports <${ontology.absolutePath}>. + +# Range -> Filter + a jvm:MemoryChannelReader. + a jvm:MemoryChannelWriter. + +# Define a filter processor. +[] + a jvm:RDFValidator; + jvm:input ; + jvm:output ; + jvm:shapes "${shapeFile.absolutePath}"; + jvm:error_is_fatal true; + jvm:print_report true. +""" + +private val pipelineFile = File.createTempFile("pipeline", "ttl").apply { writeText(pipeline) } + +class RDFValidatorTest { + @Test + fun definition() { + // Initialize pipeline. + val pipeline = Pipeline(pipelineFile) + + // Extract processor. + val processors = pipeline.processors + assertEquals(1, processors.size) + val validator = processors[0] as RDFValidator + + // Check arguments. + assertEquals(shapeFile.absolutePath, validator.getArgument("shapes")) + assertEquals(Optional.of(true), validator.getArgument("error_is_fatal")) + assertEquals(Optional.of(true), validator.getArgument("error_is_fatal")) + } + + @Test + fun conforms() { + // Setup channels. + val input = DummyReader(arrayOf(validInput.toByteArray())) + val output = DummyWriter() + + // Create a new RDFValidator instance. + val validator = + RDFValidator( + mapOf( + "shapes" to shapeFile.path, + "input" to input, + "output" to output, + "error_is_fatal" to Optional.of(true), + "print_report" to Optional.of(true), + )) + + // Execute validator. + validator.exec() + + // Check if the output is correct. + assertEquals(1, output.getValues().size) + assertEquals(validInput, output.getValues()[0].decodeToString()) + } + + @Test + fun doesNotConformAndIsFatal() { + // Setup channels. + val input = DummyReader(arrayOf(invalidInput.toByteArray())) + val output = DummyWriter() + + // Create a new RDFValidator instance. + val validator = + RDFValidator( + mapOf( + "shapes" to shapeFile.path, + "input" to input, + "output" to output, + "error_is_fatal" to Optional.of(true), + "print_report" to Optional.of(false), + )) + + // Execute validator. + assertThrows { validator.exec() } + + // Nothing has been written. + assertEquals(0, output.getValues().size) + } + + @Test + fun doesNotConform() { + // Setup channels. + val input = DummyReader(arrayOf(invalidInput.toByteArray())) + val output = DummyWriter() + + // Create a new RDFValidator instance. + val validator = + RDFValidator( + mapOf( + "shapes" to shapeFile.path, + "input" to input, + "output" to output, + "error_is_fatal" to Optional.of(false), + "print_report" to Optional.of(false), + )) + + // Execute validator. + assertDoesNotThrow { validator.exec() } + + // Nothing has been written. + assertEquals(0, output.getValues().size) + } + + @Test + fun invalidSHACL() { + // Fails during initialization of the SHACL model. + assertThrows { + RDFValidator( + mapOf( + "shapes" to invalidShapeFile.path, + "input" to DummyReader(arrayOf()), + "output" to DummyWriter(), + "error_is_fatal" to Optional.of(true), + "print_report" to Optional.of(false), + )) + } + } +} diff --git a/src/test/resources/std b/src/test/resources/std new file mode 120000 index 0000000..b8610f5 --- /dev/null +++ b/src/test/resources/std @@ -0,0 +1 @@ +../../../src/main/resources/std \ No newline at end of file