[Feature] Advanced Queries & Boolean Retrieval

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/context/QueryContext.kt

@@ -5,14 +5,35 @@ import kotlinx.serialization.Serializable
 import kotlinx.serialization.Transient
 import org.vitrivr.engine.core.model.metamodel.Schema
 
-
+/**
+ * [Context] used for queries.
+ *
+ * @author Loris Sauter
+ * @version 1.0.0
+ */
 @Serializable
 class QueryContext(
     override val local: Map<String, Map<String, String>> = emptyMap(),
     override val global: Map<String, String> = emptyMap()
 ) : Context() {
 
+    /**
+     *
+     */
+    companion object {
+        /** The key for the [QueryContext] limit parameter. */
+        const val LIMIT_KEY = "limit"
+
+        /** The key for the [QueryContext] 'fetch_descriptor' parameter. */
+        const val FETCH_DESCRIPTOR_KEY = "returnDescriptor"
+
+        /** The default for the [QueryContext] limit parameter. */
+        const val LIMIT_DEFAULT = 1000L
+    }
+
     @Transient
     @get:OpenApiIgnore
     override lateinit var schema: Schema
+
+
 }

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/blackhole/descriptors/BlackholeDescriptorReader.kt

@@ -13,7 +13,7 @@ import org.vitrivr.engine.core.model.retrievable.Retrieved
  * A [DescriptorReader] for the [BlackholeConnection].
  *
  * @author Ralph Gasser
- * @version 1.0.0
+ * @version 1.1.0
  */
 class BlackholeDescriptorReader<T : Descriptor<*>>(override val connection: BlackholeConnection, override val field: Schema.Field<*, T>) : DescriptorReader<T> {
     override fun exists(descriptorId: DescriptorId): Boolean = false

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/database/descriptor/DescriptorReader.kt

@@ -5,13 +5,14 @@ import org.vitrivr.engine.core.model.descriptor.Descriptor
 import org.vitrivr.engine.core.model.descriptor.DescriptorId
 import org.vitrivr.engine.core.model.metamodel.Analyser
 import org.vitrivr.engine.core.model.metamodel.Schema
+import org.vitrivr.engine.core.model.query.Predicate
 import org.vitrivr.engine.core.model.query.Query
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.RetrievableId
 import org.vitrivr.engine.core.model.retrievable.Retrieved
 
 /**
- * A [DescriptorReader] is an extension of a [Reader], that allows the execution of [Descriptor] specific [Query] objects.
+ * A [DescriptorReader] is an extension of a [Reader], that allows the execution of [Descriptor] specific [Predicate] objects.
  *
  * The [DescriptorReader] acts as a shim between the data base layer and vitrivr's data- and query model.
  *
@@ -65,15 +66,15 @@ interface DescriptorReader<D : Descriptor<*>> : Reader<D> {
     fun getAllForRetrievable(retrievableIds: Iterable<RetrievableId>): Sequence<D>
 
     /**
-     * Returns a [Sequence] of all [Descriptor]s [D]s that match the given [Query].
+     * Returns a [Sequence] of all [Descriptor]s [D]s that match the given [Predicate].
      *
      * @param query The [Query] that should be executed.
      * @return [Sequence] of [D].
      */
     fun query(query: Query): Sequence<D>
 
     /**
-     * Returns a [Sequence] of all [Retrieved]s that match the given [Query].
+     * Returns a [Sequence] of all [Retrieved]s that match the given [Predicate].
      *
      * Implicitly, this methods executes a [query] and then JOINS the result with the [Retrieved]s.
      *

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/AbstractRetriever.kt

@@ -10,6 +10,7 @@ import org.vitrivr.engine.core.database.descriptor.DescriptorReader
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.Descriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
+import org.vitrivr.engine.core.model.query.Predicate
 import org.vitrivr.engine.core.model.query.Query
 import org.vitrivr.engine.core.operators.retrieve.Retriever
 
@@ -22,7 +23,7 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever
  * @see [AbstractRetriever]
  *
  * @author Rahel Arnold
- * @version 1.0.1
+ * @version 1.1.0
  */
 abstract class AbstractRetriever<C : ContentElement<*>, D : Descriptor<*>>(override val field: Schema.Field<C, D>, val query: Query, val context: QueryContext) : Retriever<C, D> {
 
@@ -33,7 +34,7 @@ abstract class AbstractRetriever<C : ContentElement<*>, D : Descriptor<*>>(overr
     protected val logger: KLogger = KotlinLogging.logger {}
 
     /**
-     * Simplest implementation of the retrieval logic simply hand the [Query] to the [DescriptorReader] and emit the results.
+     * Simplest implementation of the retrieval logic simply hand the [Predicate] to the [DescriptorReader] and emit the results.
      *
      * @param scope The [CoroutineScope] to execute the resulting [Flow] in.
      */

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/averagecolor/AverageColor.kt

@@ -9,10 +9,9 @@ import org.vitrivr.engine.core.model.content.Content
 import org.vitrivr.engine.core.model.content.element.ImageContent
 import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor
 import org.vitrivr.engine.core.model.metamodel.Analyser
-import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge
 import org.vitrivr.engine.core.model.metamodel.Schema
 import org.vitrivr.engine.core.model.query.Query
-import org.vitrivr.engine.core.model.query.proximity.ProximityQuery
+import org.vitrivr.engine.core.model.query.proximity.ProximityPredicate
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.attributes.CONTENT_AUTHORS_KEY
 import org.vitrivr.engine.core.model.types.Value
@@ -76,28 +75,25 @@ class AverageColor : Analyser<ImageContent, FloatVectorDescriptor> {
      *
      * @return A new [Retriever] instance for this [Analyser]
      */
-    override fun newRetrieverForQuery(field: Schema.Field<ImageContent, FloatVectorDescriptor>, query: Query, context: QueryContext): DenseRetriever<ImageContent> {
-        require(query is ProximityQuery<*> && query.value is Value.FloatVector) { "The query is not a ProximityQuery<Value.FloatVector>." }
-        @Suppress("UNCHECKED_CAST")
-        return DenseRetriever(field, query as ProximityQuery<Value.FloatVector>, context, LinearCorrespondence(3f))
-    }
+    override fun newRetrieverForQuery(field: Schema.Field<ImageContent, FloatVectorDescriptor>, query: Query, context: QueryContext) = DenseRetriever(field, query, context, LinearCorrespondence(3f))
 
     /**
      * Generates and returns a new [DenseRetriever] instance for this [AverageColor].
      *
-     * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityQuery] that can be used to retrieve similar [ImageContent] elements.
+     * Invoking this method involves converting the provided [FloatVectorDescriptor] into a [ProximityPredicate] that can be used to retrieve similar [ImageContent] elements.
      *
      * @param field The [Schema.Field] to create an [Retriever] for.
      * @param descriptors An array of [FloatVectorDescriptor] elements to use with the [Retriever]
      * @param context The [QueryContext] to use with the [Retriever]
      */
     override fun newRetrieverForDescriptors(field: Schema.Field<ImageContent, FloatVectorDescriptor>, descriptors: Collection<FloatVectorDescriptor>, context: QueryContext): DenseRetriever<ImageContent> {
         /* Prepare query parameters. */
-        val k = context.getProperty(field.fieldName, "limit")?.toLongOrNull() ?: 1000L
-        val fetchVector = context.getProperty(field.fieldName, "returnDescriptor")?.toBooleanStrictOrNull() ?: false
+        val k = context.getProperty(field.fieldName, QueryContext.LIMIT_KEY)?.toLongOrNull() ?: QueryContext.LIMIT_DEFAULT
+        val fetchVector = context.getProperty(field.fieldName, QueryContext.FETCH_DESCRIPTOR_KEY)?.toBooleanStrictOrNull() == true
+        val predicate = ProximityPredicate(field = field, value = descriptors.first().vector, k = k, fetchVector = fetchVector)
 
         /* Return retriever. */
-        return this.newRetrieverForQuery(field, ProximityQuery(value = descriptors.first().vector, k = k, fetchVector = fetchVector), context)
+        return this.newRetrieverForQuery(field, Query(predicate), context)
     }
 
     /**

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/bool/ScalarBooleanRetriever.kt

@@ -7,15 +7,15 @@ import org.vitrivr.engine.core.features.AbstractRetriever
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.scalar.ScalarDescriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
-import org.vitrivr.engine.core.model.query.bool.BooleanQuery
+import org.vitrivr.engine.core.model.query.Query
 
 /**
  * A simple [AbstractRetriever] implementation for boolean queries on [ScalarDescriptor]s.
  *
  * @author Ralph Gasser
- * @version 1.0.0
+ * @version 1.1.0
  */
-class ScalarBooleanRetriever<C : ContentElement<*>>(field: Schema.Field<C, ScalarDescriptor<*, *>>, query: BooleanQuery, context: QueryContext) : AbstractRetriever<C, ScalarDescriptor<*, *>>(field, query, context) {
+class ScalarBooleanRetriever<C : ContentElement<*>>(field: Schema.Field<C, ScalarDescriptor<*, *>>, query: Query, context: QueryContext) : AbstractRetriever<C, ScalarDescriptor<*, *>>(field, query, context) {
     override fun toFlow(scope: CoroutineScope) = flow {
         val reader = this@ScalarBooleanRetriever.reader
         reader.queryAndJoin(this@ScalarBooleanRetriever.query).forEach {

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/bool/StructBooleanRetriever.kt

@@ -7,15 +7,15 @@ import org.vitrivr.engine.core.features.AbstractRetriever
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.struct.StructDescriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
-import org.vitrivr.engine.core.model.query.bool.BooleanQuery
+import org.vitrivr.engine.core.model.query.Query
 
 /**
  * A simple [AbstractRetriever] implementation for boolean queries on [StructDescriptor]s.
  *
  * @author Ralph Gasser
- * @version 1.0.0
+ * @version 1.1.0
  */
-class StructBooleanRetriever<C : ContentElement<*>, D : StructDescriptor<*>>(field: Schema.Field<C, D>, query: BooleanQuery, context: QueryContext) : AbstractRetriever<C, D>(field, query, context) {
+class StructBooleanRetriever<C : ContentElement<*>, D : StructDescriptor<*>>(field: Schema.Field<C, D>, query: Query, context: QueryContext) : AbstractRetriever<C, D>(field, query, context) {
     override fun toFlow(scope: CoroutineScope) = flow {
         val reader = this@StructBooleanRetriever.reader
         reader.queryAndJoin(this@StructBooleanRetriever.query).forEach {

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/dense/DenseRetriever.kt

@@ -8,7 +8,8 @@ import org.vitrivr.engine.core.math.correspondence.CorrespondenceFunction
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.vector.FloatVectorDescriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
-import org.vitrivr.engine.core.model.query.proximity.ProximityQuery
+import org.vitrivr.engine.core.model.query.Query
+import org.vitrivr.engine.core.model.query.proximity.ProximityPredicate
 import org.vitrivr.engine.core.model.retrievable.attributes.DistanceAttribute
 import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute
 
@@ -20,14 +21,13 @@ import org.vitrivr.engine.core.model.retrievable.attributes.ScoreAttribute
  * @param context The [QueryContext] used to execute the query with.
  *
  * @see [AbstractRetriever]
- * @see [ProximityQuery]
+ * @see [ProximityPredicate]
  *
  * @author Rahel Arnold
  * @author Fynn Faber
  * @version 1.0.0
  */
-class DenseRetriever<C : ContentElement<*>>(field: Schema.Field<C, FloatVectorDescriptor>, query: ProximityQuery<*>, context: QueryContext, val correspondence: CorrespondenceFunction) :
-    AbstractRetriever<C, FloatVectorDescriptor>(field, query, context) {
+class DenseRetriever<C : ContentElement<*>>(field: Schema.Field<C, FloatVectorDescriptor>, query: Query, context: QueryContext, val correspondence: CorrespondenceFunction) : AbstractRetriever<C, FloatVectorDescriptor>(field, query, context) {
     override fun toFlow(scope: CoroutineScope) = flow {
         this@DenseRetriever.reader.queryAndJoin(this@DenseRetriever.query).forEach {
             val distance = it.filteredAttribute<DistanceAttribute>()

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/fulltext/FulltextRetriever.kt

@@ -5,12 +5,12 @@ import org.vitrivr.engine.core.features.AbstractRetriever
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.scalar.TextDescriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
-import org.vitrivr.engine.core.model.query.fulltext.SimpleFulltextQuery
+import org.vitrivr.engine.core.model.query.Query
 
 /**
- * An implementation of a [Retriever], that executes fulltext queries.
+ * An implementation of a [AbstractRetriever], that executes fulltext queries.
  *
  * @author Ralph Gasser, Fynn Faber
  * @version 1.2.0
  */
-class FulltextRetriever<C: ContentElement<*>>(field: Schema.Field<C, TextDescriptor>, query: SimpleFulltextQuery, context: QueryContext) : AbstractRetriever<C, TextDescriptor>(field, query, context)
+class FulltextRetriever<C : ContentElement<*>>(field: Schema.Field<C, TextDescriptor>, query: Query, context: QueryContext) : AbstractRetriever<C, TextDescriptor>(field, query, context)

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/exif/ExifMetadata.kt

@@ -6,10 +6,8 @@ import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.Attribute
 import org.vitrivr.engine.core.model.descriptor.struct.AnyMapStructDescriptor
 import org.vitrivr.engine.core.model.metamodel.Analyser
-import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge
 import org.vitrivr.engine.core.model.metamodel.Schema
 import org.vitrivr.engine.core.model.query.Query
-import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.attributes.CONTENT_AUTHORS_KEY
 import org.vitrivr.engine.core.model.types.Type
@@ -55,13 +53,15 @@ class ExifMetadata : Analyser<ContentElement<*>, AnyMapStructDescriptor> {
     override fun newExtractor(field: Schema.Field<ContentElement<*>, AnyMapStructDescriptor>, input: Operator<Retrievable>, context: IndexContext) = ExifMetadataExtractor(input, this, context[field.fieldName, CONTENT_AUTHORS_KEY]?.split(",")?.toSet(), field)
 
     /**
+     * Generates and returns a new [ExifMetadataExtractor] instance for this [ExifMetadata].
+     *
+     * @param field The [Schema.Field] to create an [Retriever] for.
+     * @param query The [Query] that acts as input to the new [Retriever].
+     * @param context The [QueryContext] to use with the [Retriever].
      *
+     * @return A new [ExifMetadataRetriever] instance for this [Analyser]
      */
-    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, AnyMapStructDescriptor>, query: Query, context: QueryContext): Retriever<ContentElement<*>, AnyMapStructDescriptor> {
-        require(field.analyser == this) { "Field type is incompatible with analyser. This is a programmer's error!" }
-        require(query is SimpleBooleanQuery<*>) { "Query is not a Query." }
-        return ExifMetadataRetriever(field, query, context)
-    }
+    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, AnyMapStructDescriptor>, query: Query, context: QueryContext) = ExifMetadataRetriever(field, query, context)
 
     /**
      *

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadata.kt

@@ -5,14 +5,11 @@ import org.vitrivr.engine.core.context.QueryContext
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor
 import org.vitrivr.engine.core.model.metamodel.Analyser
-import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge
 import org.vitrivr.engine.core.model.metamodel.Schema
 import org.vitrivr.engine.core.model.query.Query
-import org.vitrivr.engine.core.model.query.bool.SimpleBooleanQuery
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.attributes.CONTENT_AUTHORS_KEY
 import org.vitrivr.engine.core.operators.Operator
-import org.vitrivr.engine.core.operators.retrieve.Retriever
 
 /**
  * Implementation of the [FileSourceMetadata] [Analyser], which derives metadata information from file-based retrievables
@@ -63,11 +60,7 @@ class FileSourceMetadata : Analyser<ContentElement<*>, FileSourceMetadataDescrip
      *
      * @return [FileSourceMetadataRetriever]
      */
-    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, FileSourceMetadataDescriptor>, query: Query, context: QueryContext): Retriever<ContentElement<*>, FileSourceMetadataDescriptor> {
-        require(field.analyser == this) { "Field type is incompatible with analyser. This is a programmer's error!" }
-        require(query is SimpleBooleanQuery<*>) { "Query is not a Query." }
-        return FileSourceMetadataRetriever(field, query, context)
-    }
+    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, FileSourceMetadataDescriptor>, query: Query, context: QueryContext) = FileSourceMetadataRetriever(field, query, context)
 
     /**
      * [FileSourceMetadataRetriever] Cannot derive a [FileSourceMetadataRetriever] from content.

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/file/FileSourceMetadataRetriever.kt

@@ -5,7 +5,7 @@ import org.vitrivr.engine.core.features.AbstractRetriever
 import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor
 import org.vitrivr.engine.core.model.metamodel.Schema
-import org.vitrivr.engine.core.model.query.bool.BooleanQuery
+import org.vitrivr.engine.core.model.query.Query
 import org.vitrivr.engine.core.operators.retrieve.Retriever
 
 /**
@@ -14,4 +14,4 @@ import org.vitrivr.engine.core.operators.retrieve.Retriever
  * @author Ralph Gasser
  * @version 1.0.0
  */
-class FileSourceMetadataRetriever(field: Schema.Field<ContentElement<*>, FileSourceMetadataDescriptor>, query: BooleanQuery, context: QueryContext) : AbstractRetriever<ContentElement<*>, FileSourceMetadataDescriptor>(field, query, context)
+class FileSourceMetadataRetriever(field: Schema.Field<ContentElement<*>, FileSourceMetadataDescriptor>, query: Query, context: QueryContext) : AbstractRetriever<ContentElement<*>, FileSourceMetadataDescriptor>(field, query, context)

vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/features/metadata/source/video/VideoSourceMetadata.kt

@@ -9,10 +9,8 @@ import org.vitrivr.engine.core.model.content.element.ContentElement
 import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.FileSourceMetadataDescriptor
 import org.vitrivr.engine.core.model.descriptor.struct.metadata.source.VideoSourceMetadataDescriptor
 import org.vitrivr.engine.core.model.metamodel.Analyser
-import org.vitrivr.engine.core.model.metamodel.Analyser.Companion.merge
 import org.vitrivr.engine.core.model.metamodel.Schema
 import org.vitrivr.engine.core.model.query.Query
-import org.vitrivr.engine.core.model.query.bool.BooleanQuery
 import org.vitrivr.engine.core.model.retrievable.Retrievable
 import org.vitrivr.engine.core.model.retrievable.attributes.CONTENT_AUTHORS_KEY
 import org.vitrivr.engine.core.operators.Operator
@@ -67,11 +65,7 @@ class VideoSourceMetadata : Analyser<ContentElement<*>, VideoSourceMetadataDescr
      *
      * @return [VideoSourceMetadataRetriever]
      */
-    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, VideoSourceMetadataDescriptor>, query: Query, context: QueryContext): VideoSourceMetadataRetriever {
-        require(field.analyser == this) { "Field type is incompatible with analyser. This is a programmer's error!" }
-        require(query is BooleanQuery) { "Query is not a Boolean query." }
-        return VideoSourceMetadataRetriever(field, query, context)
-    }
+    override fun newRetrieverForQuery(field: Schema.Field<ContentElement<*>, VideoSourceMetadataDescriptor>, query: Query, context: QueryContext) = VideoSourceMetadataRetriever(field, query, context)
 
     /**
      * [FileSourceMetadataRetriever] Cannot derive a [VideoSourceMetadataRetriever] from content.