Merge branch 'master' into dev

config.json

@@ -1,8 +1,3 @@
 {
-  "root": "./cottontaildb",
-  "statistics" : {
-    "threshold" : 1.0,
-    "probability" : 0.5,
-    "randomGeneratorName": "L32X64MixRandom"
-  }
+  "root": "/cottontaildb-data"
 }

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/Playground.kt

@@ -1,15 +1,13 @@
 package org.vitrivr.cottontail
 
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.builtins.ListSerializer
-import kotlinx.serialization.json.DecodeSequenceMode
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.decodeToSequence
-import kotlinx.serialization.json.encodeToStream
-import java.nio.file.Files
-import java.nio.file.Paths
-import java.nio.file.StandardOpenOption
-import java.util.*
+import io.grpc.ManagedChannel
+import io.grpc.ManagedChannelBuilder
+import org.vitrivr.cottontail.client.SimpleClient
+import org.vitrivr.cottontail.client.language.basics.Direction
+import org.vitrivr.cottontail.client.language.basics.Distances
+import org.vitrivr.cottontail.client.language.dql.Query
+import org.vitrivr.cottontail.core.database.Name
+import org.vitrivr.cottontail.core.values.generators.FloatVectorValueGenerator
 
 /**
  *
@@ -18,44 +16,24 @@ import java.util.*
  */
 
 fun main(args: Array<String>) {
-    val path = Paths.get("/Users/rgasser/Downloads/package/warren.VBSLHE.descriptor_file.0.json")
-    val json = Json { prettyPrint = true }
-    Files.newInputStream(path, StandardOpenOption.READ).use { input ->
-        val buffer = LinkedList<FileDescriptor>()
-        var index = 0
-        val descriptors = Json.decodeToSequence(input, FileDescriptor.serializer(), DecodeSequenceMode.ARRAY_WRAPPED)
-        for (descriptor in descriptors) {
-            buffer.add(descriptor)
-            if (buffer.size == 50_000) {
-                write(index++, buffer)
-                buffer.clear()
-            }
-        }
-        write(index, buffer)
-    }
-}
 
-/**
- *
- */
-fun write(index: Int, buffer: LinkedList<FileDescriptor>) {
-    val path = Paths.get("/Users/rgasser/Downloads/package/out/warren.VBSLHE.descriptor_file.$index.json")
-    Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE).use { output ->
-        Json.encodeToStream(ListSerializer(FileDescriptor.serializer()), buffer, output)
-    }
-}
+    val channel: ManagedChannel = ManagedChannelBuilder.forAddress("127.0.0.1", 1865)
+        .enableFullStreamDecompression()
+        .usePlaintext()
+        .build()
+
+    val client = SimpleClient(channel)
 
-@Serializable
-data class Descriptor(val descriptorId: String, val retrievableId: String, val descriptor: FloatArray)
 
-@Serializable
-data class Retrievable(val retrievableId: String, val type: String)
+    val query = Query(Name.EntityName.create("vitrivr", "descriptor_clip"))
+        .select("retrievableId")
+        .distance("descriptor", FloatVectorValueGenerator.zero(512), Distances.L2, "distance")
+        .order("distance", Direction.ASC)
+        .limit(1000)
 
-@Serializable
-data class Relationship(val objectId: String, val subjectId: String, val predicate: String)
 
-@Serializable
-data class FileDescriptor(val descriptorId: String = UUID.randomUUID().toString(), val retrievableId: String, val size: Int, val path: String) {}
+    val results = client.query(query).forEach {
+        println(it)
+    }
+}
 
-@Serializable
-data class TimeDescriptor(val descriptorId: String, val retrievableId: String, val start: Long, val end: Long)

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/CreateEntityCommand.kt

@@ -148,7 +148,7 @@ class CreateEntityCommand(client: SimpleClient) : AbstractEntityCommand(client,
             "LONG_VECTOR",
             "FLOAT_VECTOR",
             "DOUBLE_VECTOR",
-            "BOOL_VECTOR",
+            "BOOLEAN_VECTOR",
             "COMPLEX32_VECTOR",
             "COMPLEX64_VECTOR" -> this.terminal.prompt("\rColumn lengths (i.e., number of entries for vectors)")?.toIntOrNull() ?: 1
             else -> -1

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/ImportDataCommand.kt

@@ -59,61 +59,59 @@ class ImportDataCommand(client: SimpleClient) : AbstractEntityCommand(client, na
     override fun exec() {
         /* Read schema and prepare Iterator. */
         val schema = this.client.readSchema(this.entityName).toTypedArray()
-        val data: Sequence<Tuple> = when(format) {
-            Format.CBOR -> Cbor.decodeFromByteArray(ListSerializer(schema.serializer()), Files.readAllBytes(this.input)).asSequence()
-            Format.JSON -> Files.newInputStream(this.input).use {
-                Json.decodeToSequence(it, schema.serializer(), DecodeSequenceMode.ARRAY_WRAPPED)
+        Files.newInputStream(this.input).use { input ->
+            val data: Sequence<Tuple> = when(format) {
+                Format.CBOR -> Cbor.decodeFromByteArray(ListSerializer(schema.serializer()), input.readAllBytes()).asSequence()
+                Format.JSON -> Json.decodeToSequence(input, schema.serializer(), DecodeSequenceMode.ARRAY_WRAPPED)
+                Format.CSV -> Csv.decodeFromString(ListSerializer(schema.descriptionSerializer()), input.readAllBytes().toString()).asSequence()
             }
-            Format.CSV ->Files.newInputStream(this.input).use {
-                Csv.decodeFromString(ListSerializer(schema.descriptionSerializer()), it.readAllBytes().toString())
-            }.asSequence()
-        }
-
-        /** Begin transaction (if single transaction option has been set). */
-        val txId = if (this.singleTransaction) {
-            this.client.begin()
-        } else {
-            null
-        }
 
-        try {
-            /* Prepare batch insert message. */
-            val batchedInsert = BatchInsert(this.entityName.fqn)
-            if (txId != null) {
-                batchedInsert.txId(txId)
+            /** Begin transaction (if single transaction option has been set). */
+            val txId = if (this.singleTransaction) {
+                this.client.begin()
+            } else {
+                null
             }
-            batchedInsert.columns(*schema.map { it.name.simple }.toTypedArray())
-            var count = 0L
-            val duration = measureTime {
-                for (t in data) {
-                    if (!batchedInsert.values(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) {
-                        /* Execute insert... */
-                        client.insert(batchedInsert)
 
-                        /* ... now clear and append. */
-                        batchedInsert.clear()
-                        if (!batchedInsert.any(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) {
-                            throw IllegalArgumentException("The appended data is too large for a single message.")
+            try {
+                /* Prepare batch insert message. */
+                val batchedInsert = BatchInsert(this.entityName.fqn)
+                if (txId != null) {
+                    batchedInsert.txId(txId)
+                }
+                batchedInsert.columns(*schema.map { it.name.simple }.toTypedArray())
+                var count = 0L
+                val duration = measureTime {
+                    for (t in data) {
+                        if (!batchedInsert.values(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) {
+                            /* Execute insert... */
+                            client.insert(batchedInsert)
+
+                            /* ... now clear and append. */
+                            batchedInsert.clear()
+                            if (!batchedInsert.any(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) {
+                                throw IllegalArgumentException("The appended data is too large for a single message.")
+                            }
                         }
+                        count += 1
                     }
-                    count += 1
-                }
 
-                /** Insert remainder. */
-                if (batchedInsert.count() > 0) {
-                    this.client.insert(batchedInsert)
-                }
+                    /** Insert remainder. */
+                    if (batchedInsert.count() > 0) {
+                        this.client.insert(batchedInsert)
+                    }
 
-                /** Commit transaction, if single transaction option has been set. */
-                if (txId != null) {
-                    this.client.commit(txId)
+                    /** Commit transaction, if single transaction option has been set. */
+                    if (txId != null) {
+                        this.client.commit(txId)
+                    }
                 }
+                println("Importing $count entries into $entityName took $duration.")
+            } catch (e: Throwable) {
+                /** Rollback transaction, if single transaction option has been set. */
+                if (txId != null) this.client.rollback(txId)
+                println("Importing entries into $entityName failed due to error: ${e.message}")
             }
-            println("Importing $count entries into $entityName took $duration.")
-        } catch (e: Throwable) {
-            /** Rollback transaction, if single transaction option has been set. */
-            if (txId != null) this.client.rollback(txId)
-            println("Importing entries into $entityName failed due to error: ${e.message}")
         }
     }
 }

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/query/PreviewEntityCommand.kt

@@ -46,7 +46,7 @@ class PreviewEntityCommand(client: SimpleClient): AbstractQueryCommand(client, n
 
     override fun exec() {
         /* Prepare query. */
-        val query = Query(this.entityName.toString()).select("*").limit(this.limit).skip(this.skip)
+        val query = Query(this.entityName.toString()).select("*").disallowParallelism().limit(this.limit).skip(this.skip)
         if (this.toFile) {
             this.executeAndExport(query)
         } else {

cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/data/Dumper.kt

@@ -219,7 +219,7 @@ abstract class Dumper(protected val client: SimpleClient, protected val output:
 
             /* Start dumping the entity in batches. */
             val buffer = mutableListOf<Tuple>()
-            val results = this.client.query(Query(entity).txId(this.txId))
+            val results = this.client.query(Query(entity).txId(this.txId).disallowParallelism())
             var dumped = 0L
             var batch = 0L
 

cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/client/language/basics/expression/Literal.kt

@@ -4,12 +4,13 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import org.vitrivr.cottontail.core.values.*
 import org.vitrivr.cottontail.grpc.CottontailGrpc
+import java.util.*
 
 /**
  * A [Literal] value [Expression].
  *
  * @author Ralph Gasser
- * @version 1.0.0
+ * @version 1.1.0
  */
 @Serializable
 @SerialName("Literal")
@@ -22,6 +23,8 @@ data class Literal(val value: PublicValue): Expression() {
     constructor(value: Float): this(FloatValue(value))
     constructor(value: Double): this(DoubleValue(value))
     constructor(value: String): this(StringValue(value))
+    constructor(value: UUID): this(UuidValue(value))
+    constructor(value: Date): this(DateValue(value))
     constructor(value: BooleanArray): this(BooleanVectorValue(value))
     constructor(value: IntArray): this(IntVectorValue(value))
     constructor(value: LongArray): this(LongVectorValue(value))

cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/client/language/basics/expression/ValueList.kt

@@ -0,0 +1,45 @@
+package org.vitrivr.cottontail.client.language.basics.expression
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import org.vitrivr.cottontail.core.values.*
+import org.vitrivr.cottontail.grpc.CottontailGrpc
+import java.util.*
+
+/**
+ * A list of [Literal] values [Expression]. Mainly used for IN queries.
+ *
+ * @author Ralph Gasser
+ * @version 1.1.0
+ */
+@Serializable
+@SerialName("ValueList")
+class ValueList(val value: Array<PublicValue>): Expression() {
+    constructor(list: List<Any>): this(when (list.first()) {
+        is Boolean -> list.filterIsInstance<Boolean>().map { BooleanValue(it) }
+        is Byte -> list.filterIsInstance<Byte>().map { ByteValue(it) }
+        is Short -> list.filterIsInstance<Short>().map { ShortValue(it) }
+        is Int -> list.filterIsInstance<Int>().map { IntValue(it) }
+        is Long -> list.filterIsInstance<Long>().map { LongValue(it) }
+        is Float -> list.filterIsInstance<Float>().map { FloatValue(it) }
+        is Double -> list.filterIsInstance<Double>().map { DoubleValue(it) }
+        is Date -> list.filterIsInstance<Date>().map { DateValue(it) }
+        is String -> list.filterIsInstance<String>().map { StringValue(it) }
+        is UUID -> list.filterIsInstance<UUID>().map { UuidValue(it) }
+        is BooleanArray -> list.filterIsInstance<BooleanArray>().map { BooleanVectorValue(it) }
+        is IntArray -> list.filterIsInstance<IntArray>().map { IntVectorValue(it) }
+        is LongArray -> list.filterIsInstance<LongArray>().map { LongVectorValue(it) }
+        is FloatArray -> list.filterIsInstance<FloatArray>().map { FloatVectorValue(it) }
+        is DoubleArray -> list.filterIsInstance<DoubleArray>().map { DoubleVectorValue(it) }
+        is PublicValue -> list.filterIsInstance<PublicValue>()
+        else -> throw IllegalArgumentException("Cannot create ValueList from list of type ${list.first().javaClass.simpleName}.")
+    }.toTypedArray())
+
+    override fun toGrpc(): CottontailGrpc.Expression {
+        val builder = CottontailGrpc.Expression.newBuilder()
+        for (data in this.value) {
+            builder.literalListBuilder.addLiteral(data.toGrpc())
+        }
+        return builder.build()
+    }
+}