diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/math/Half.kt similarity index 99% rename from cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt rename to cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/math/Half.kt index ec04faa84..f04db5cfc 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/math/Half.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/math/Half.kt @@ -18,10 +18,10 @@ // Operators +, *, / based on http://half.sourceforge.net/ by Christian Rau // and licensed under MIT -package org.vitrivr.cottontail.utilities.math +package org.vitrivr.cottontail.math -import org.vitrivr.cottontail.utilities.math.Half.Companion.POSITIVE_INFINITY -import org.vitrivr.cottontail.utilities.math.Half.Companion.POSITIVE_ZERO +import org.vitrivr.cottontail.math.Half.Companion.POSITIVE_INFINITY +import org.vitrivr.cottontail.math.Half.Companion.POSITIVE_ZERO /** * Converts the specified double-precision float value into a diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt index 95b402a80..50c7be602 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/core/values/tablets/HalfVectorTablet.kt @@ -3,7 +3,7 @@ package org.vitrivr.cottontail.core.values.tablets import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatVectorValue import org.vitrivr.cottontail.core.values.HalfVectorValue -import org.vitrivr.cottontail.utilities.math.Half +import org.vitrivr.cottontail.math.Half /** * A [AbstractTablet] implementation for [FloatVectorValue]s (half-precision). diff --git a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt index a0d0d16d7..c188aa2dc 100644 --- a/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt +++ b/cottontaildb-core/src/main/kotlin/org/vitrivr/cottontail/utilities/hashing/ValueFunnel.kt @@ -70,6 +70,10 @@ object ValueFunnel: Funnel { into.putInt(value.logicalSize) value.data.forEach { into.putShort(it) } } + is HalfVectorValue -> { + into.putInt(from.logicalSize) + value.data.forEach { into.putFloat(it) } + } is FloatVectorValue -> { into.putInt(value.logicalSize) value.data.forEach { into.putFloat(it) } diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt index 90cb366c3..d787950cf 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/tuples/TupleSerializer.kt @@ -1,398 +1,90 @@ package org.vitrivr.cottontail.storage.serializers.tuples +import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.LongBinding +import jetbrains.exodus.util.ByteArraySizedInputStream +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.database.ColumnDef -import org.vitrivr.cottontail.core.toByte import org.vitrivr.cottontail.core.tuple.StandaloneTuple import org.vitrivr.cottontail.core.tuple.Tuple -import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.types.Value -import org.vitrivr.cottontail.core.values.* -import org.vitrivr.cottontail.utilities.math.Half +import org.vitrivr.cottontail.storage.serializers.SerializerFactory +import org.vitrivr.cottontail.storage.serializers.values.ValueSerializer +import java.io.ByteArrayInputStream import java.nio.ByteBuffer -import java.nio.charset.Charset +import java.util.* /** * A facility that can be used to serialize and deserialize a [Tuple] from/to a [ByteBuffer] * * @author Ralph Gasser - * @version 1.0.0 + * @version 3.0.0 */ class TupleSerializer(val schema: Array>) { - /** - * Calculates the size of the [ByteBuffer] required to hold the provided [Tuple] - * - * @param tuple The [Tuple] to write. - * @return Size of [ByteBuffer] in bytes. - */ - fun sizeOf(tuple: Tuple): Int { - var size = Long.SIZE_BYTES /* Size of tupleId ID + number of columns */ - for (i in 0 until this.schema.size) { - size += when(val value = tuple[i] as? PublicValue) { - is BooleanValue, - is ByteValue -> Byte.SIZE_BYTES - is BooleanVectorValue -> TODO() - is Complex32Value -> 2 * Float.SIZE_BYTES - is Complex32VectorValue -> 2 * value.logicalSize * Float.SIZE_BYTES - is Complex64Value -> 2 * Double.SIZE_BYTES - is Complex64VectorValue -> 2 * value.logicalSize * Double.SIZE_BYTES - is DateValue -> Long.SIZE_BYTES - is DoubleValue -> Double.SIZE_BYTES - is DoubleVectorValue -> value.logicalSize * Float.SIZE_BYTES - is FloatValue -> Float.SIZE_BYTES - is FloatVectorValue -> value.logicalSize * Float.SIZE_BYTES - is HalfVectorValue -> value.logicalSize * Short.SIZE_BYTES - is IntValue -> Int.SIZE_BYTES - is IntVectorValue -> value.logicalSize * Int.SIZE_BYTES - is LongValue -> Long.SIZE_BYTES - is LongVectorValue -> value.logicalSize * Long.SIZE_BYTES - is ShortValue -> Short.SIZE_BYTES - is ShortVectorValue -> value.logicalSize * Short.SIZE_BYTES - is ByteStringValue -> value.logicalSize - is StringValue -> value.value.length * Char.SIZE_BYTES - is UuidValue -> 2 * Long.SIZE_BYTES - null -> when(this.schema[i].type) { - Types.Boolean, - Types.Byte -> Byte.SIZE_BYTES - Types.Complex32, - is Types.Complex32Vector -> 2 * Float.SIZE_BYTES - Types.Complex64, - is Types.Complex64Vector-> 2 * Double.SIZE_BYTES - Types.Double, - is Types.DoubleVector -> Double.SIZE_BYTES - Types.Float, - is Types.FloatVector -> Float.SIZE_BYTES - Types.Long, - Types.Date, - is Types.LongVector -> Long.SIZE_BYTES - Types.Short -> Short.SIZE_BYTES - Types.ByteString, - Types.String, - Types.Uuid, - Types.Int, - is Types.IntVector -> Int.SIZE_BYTES - is Types.ShortVector -> Short.SIZE_BYTES - is Types.HalfVector -> Short.SIZE_BYTES - is Types.BooleanVector -> TODO() - } - } - } - return size - } - /** - * Writes a [Tuple] to a new [ByteBuffer]. - * - * @param tuple The [Tuple] to write. - * @return [ByteBuffer] - */ - fun toByteBuffer(tuple: Tuple) = toByteBuffer(tuple, ByteBuffer.allocate(sizeOf(tuple))) + /** [List] if [ValueSerializer]s used by this [TupleSerializer]. */ + private val serializers: List> = this.schema.map { SerializerFactory.value(it.type) as ValueSerializer } /** - * Writes a [Tuple] to the provided [ByteBuffer]. + * Converts a [ByteIterable] to a [Value]. * - * @param tuple The [Tuple] to write. - * @return [ByteBuffer] + * @param entry The [ByteIterable] to convert. + * @return The resulting [Value]. */ - fun toByteBuffer(tuple: Tuple, buffer: ByteBuffer): ByteBuffer { - buffer.mark() - require(schema.contentDeepEquals(tuple.columns)) { "Tuple is not compatible with provided schema." } - buffer.putLong(tuple.tupleId) - for (i in 0 until this.schema.size) { - val value = tuple[i] as? PublicValue - this.writeValue(value, this.schema[i].type, buffer) - } - return buffer.reset() - } + fun fromEntry(entry: ByteIterable) = this.read(ByteArraySizedInputStream(entry.bytesUnsafe, 0, entry.length)) /** - * Reads a [Tuple] from the provided [ByteBuffer]. + * Converts a [Value] to a [ByteIterable]. * - * @param buffer The [ByteBuffer] to read [Tuple] from. - * @return [Tuple] + * @param value The [Value] to convert. + * @return The resulting [ByteIterable]. */ - fun fromByteBuffer(buffer: ByteBuffer): Tuple { - val tupleId = buffer.long - val values = Array(this.schema.size) { - val column = this.schema[it] - if (column.nullable) { - this.readNullableValue(column.type, buffer) - } else { - this.readNonNullableValue(column.type, buffer) - } - } - return StandaloneTuple(tupleId, this.schema, values) + fun toEntry(value: Tuple): ByteIterable { + val output = LightOutputStream(this.schema.sumOf { it.type.physicalSize }) + this.write(output, value) + return output.asArrayByteIterable() } /** - * Writs a [PublicValue] (or null) of the given [Types] to the [ByteBuffer]. + * Reads a [Tuple] from a [ByteArrayInputStream]. * - * @param value The [PublicValue] to write. - * @param type The [Types] of the [PublicValue] (in case it is null). - * @param buffer The [ByteBuffer] to write to. + * @param input The [ByteArrayInputStream] to read. + * @return The resulting [Tuple]. */ - private fun writeValue(value: PublicValue?, type: Types<*>, buffer: ByteBuffer) { - when (value) { - is BooleanValue -> buffer.put(value.value.toByte()) - is ByteValue -> buffer.put(value.value) - is ShortValue -> buffer.putShort(value.value) - is IntValue -> buffer.putInt(value.value) - is LongValue -> buffer.putLong(value.value) - is FloatValue -> buffer.putFloat(value.value) - is DoubleValue -> buffer.putDouble(value.value) - is DateValue -> buffer.putLong(value.value) - is Complex32Value -> { - buffer.putFloat(value.data[0]) - buffer.putFloat(value.data[1]) - } - - is Complex64Value -> { - buffer.putDouble(value.data[0]) - buffer.putDouble(value.data[0]) - } - - is ByteStringValue -> { - buffer.putInt(value.value.size) - buffer.put(value.value) - } - - is StringValue -> { - val bytes = value.value.toByteArray(Charset.defaultCharset()) - buffer.putInt(bytes.size) - buffer.put(bytes) - } - is UuidValue -> { - buffer.putLong(value.value.leastSignificantBits) - buffer.putLong(value.value.mostSignificantBits) - } - - is Complex32VectorValue -> value.data.forEach { buffer.putFloat(it) } - is Complex64VectorValue -> value.data.forEach { buffer.putDouble(it) } - is BooleanVectorValue -> TODO() - is DoubleVectorValue -> value.data.forEach { buffer.putDouble(it) } - is FloatVectorValue -> value.data.forEach { buffer.putFloat(it) } - is HalfVectorValue -> value.data.forEach { buffer.putShort(Half(it).toShort()) } - is IntVectorValue -> value.data.forEach { buffer.putInt(it) } - is LongVectorValue -> value.data.forEach { buffer.putLong(it) } - is ShortVectorValue -> value.data.forEach { buffer.putShort(it) } - null -> when (type) { - Types.Boolean, - is Types.BooleanVector, - Types.Byte -> buffer.put(Byte.MIN_VALUE) - - Types.Short, - is Types.ShortVector -> buffer.putShort(Short.MIN_VALUE) - Types.Int, - is Types.IntVector -> buffer.putInt(Int.MIN_VALUE) - - Types.Long, - Types.Date, - is Types.LongVector -> buffer.putLong(Long.MIN_VALUE) - - Types.Double, - is Types.DoubleVector -> buffer.putDouble(Double.MIN_VALUE) - - Types.Float, - is Types.FloatVector -> buffer.putFloat(Float.MIN_VALUE) - - Types.Complex32, - is Types.Complex32Vector -> { - buffer.putFloat(Float.MIN_VALUE) - buffer.putFloat(Float.MIN_VALUE) - } - - Types.Complex64, - is Types.Complex64Vector -> { - buffer.putDouble(Double.MIN_VALUE) - buffer.putDouble(Double.MIN_VALUE) - } - - is Types.HalfVector -> buffer.putShort(Half.MIN_VALUE.toShort()) - - Types.ByteString, - Types.String, - Types.Uuid -> buffer.putInt(-1) - - } - - - } - } - - /** - * Reads a non-nullable [Value] of the given [Types] from the [ByteBuffer]. - * - * @param type [Types] of the [Value] - * @param buffer The [ByteBuffer] to read from. - * @return [Value] - */ - private fun readNullableValue(type: Types<*>, buffer: ByteBuffer): Value? = when(type) { - Types.Boolean -> { - val value = buffer.get() - if (value == Byte.MIN_VALUE) { null } else { BooleanValue(value == 0.toByte()) } - } - Types.Byte -> { - val value = buffer.get() - if (value == Byte.MIN_VALUE) { null } else { ByteValue(value) } - } - Types.Complex32 -> { - val real = buffer.float - val imaginary = buffer.float - if (real == Float.MIN_VALUE && imaginary == Float.MIN_VALUE) { null } else { Complex32Value(real, imaginary) } - } - Types.Complex64 -> { - val real = buffer.double - val imaginary = buffer.double - if (real == Double.MIN_VALUE && imaginary == Double.MIN_VALUE) { null } else { Complex64Value(real, imaginary) } - } - Types.Double -> { - val value = buffer.double - if (value == Double.MIN_VALUE) { null } else { DoubleValue(value) } - } - Types.Float -> { - val value = buffer.float - if (value == Float.MIN_VALUE) { null } else { FloatValue(value) } - } - Types.Int -> { - val value = buffer.int - if (value == Int.MIN_VALUE) { null } else { IntValue(value) } - } - Types.Long -> { - val value = buffer.long - if (value == Long.MIN_VALUE) { null } else { LongValue(value) } - } - Types.Short -> { - val value = buffer.short - if (value == Short.MIN_VALUE) { null } else { ShortValue(value) } - } - Types.Date -> { - val value = buffer.long - if (value == Long.MIN_VALUE) { null } else { DateValue(value) } - } - Types.ByteString -> { - val length = buffer.int /* Variable length. */ - if (length == -1) { null } else { ByteStringValue(ByteArray(length) { buffer.get() }) } - } - Types.String -> { - val length = buffer.int /* Variable length. */ - if (length == -1) { null } else { StringValue(ByteArray(length) { buffer.get() }.toString(Charset.defaultCharset())) } - } - Types.Uuid -> { - val leastSignificant = buffer.long - val mostSignificant = buffer.long - UuidValue(leastSignificant, mostSignificant) - } - is Types.BooleanVector -> TODO() - is Types.Complex32Vector -> { - buffer.mark() - if (buffer.double == Double.MIN_VALUE) { - null - } else { - buffer.reset() - Complex32VectorValue(FloatArray(2*type.logicalSize) { buffer.float }) - } - } - is Types.Complex64Vector -> { - buffer.mark() - if (buffer.double == Double.MIN_VALUE) { - null - } else { - buffer.reset() - Complex64VectorValue(DoubleArray(2*type.logicalSize) { buffer.double }) - } - } - is Types.DoubleVector -> { - buffer.mark() - if (buffer.double == Double.MIN_VALUE) { - null - } else { - buffer.reset() - DoubleVectorValue(DoubleArray(type.logicalSize) { buffer.double }) - } - } - is Types.FloatVector -> { - buffer.mark() - if (buffer.float == Float.MIN_VALUE) { - null + fun read(input: ByteArrayInputStream): Tuple { + val longs = ((this.schema.size-1) shr 6) + 1 + val array = LongArray(longs) { LongBinding.BINDING.readObject(input) } + val bitset = BitSet.valueOf(array) + + val values = Array(this.schema.size) { + if (bitset[it]) { + this.serializers[it].read(input) } else { - buffer.reset() - FloatVectorValue(FloatArray(type.logicalSize) { buffer.float }) - } - } - is Types.IntVector -> { - buffer.mark() - if (buffer.int == Int.MIN_VALUE) { null - } else { - buffer.reset() - IntVectorValue(IntArray(type.logicalSize) { buffer.int }) - } - } - is Types.LongVector -> { - buffer.mark() - if (buffer.long == Long.MIN_VALUE) { - null - } else { - buffer.reset() - LongVectorValue(LongArray(type.logicalSize) { buffer.long }) } } - is Types.ShortVector -> { - buffer.mark() - if (buffer.short == Short.MIN_VALUE) { - null - } else { - buffer.reset() - ShortVectorValue(ShortArray(type.logicalSize) { buffer.short }) - } - } - - is Types.HalfVector -> { - buffer.mark() - if (buffer.short == Half.MIN_VALUE.toShort()) { - null - } else { - buffer.reset() - FloatVectorValue(FloatArray(type.logicalSize) { Half(buffer.short.toUShort()).toFloat() }) - } - } + return StandaloneTuple(0L, this.schema, values) } /** - * Reads a non-nullable [Value] of the given [Types] from the [ByteBuffer]. + * Writes a [Tuple] to a [LightOutputStream]. * - * @param type [Types] of the [Value] - * @param buffer The [ByteBuffer] to read from. - * @return [Value] + * @param output The [LightOutputStream] to read. */ - private fun readNonNullableValue(type: Types<*>, buffer: ByteBuffer): Value = when(type) { - Types.Boolean -> BooleanValue(buffer.get() == 0.toByte()) - Types.Byte -> ByteValue(buffer.get()) - Types.Complex32 -> Complex32Value(buffer.float, buffer.float) - Types.Complex64 -> Complex64Value(buffer.double, buffer.double) - Types.Double -> DoubleValue(buffer.double) - Types.Float -> FloatValue(buffer.float) - Types.Int -> IntValue(buffer.int) - Types.Long -> LongValue(buffer.long) - Types.Short -> ShortValue(buffer.short) - Types.Date -> DateValue(buffer.long) - Types.ByteString -> { - val length = buffer.int /* Variable length. */ - ByteStringValue(ByteArray(length) { buffer.get() }) + fun write(output: LightOutputStream, value: Tuple) { + /* Write null bits. */ + val bits = BitSet(this.schema.size) + for ((i, v) in value.values().withIndex()) { + bits.set(i, v != null) } - Types.String -> { - val length = buffer.int /* Variable length. */ - StringValue(ByteArray(length) { buffer.get() }.toString(Charset.defaultCharset())) + bits.toLongArray().forEach { LongBinding.BINDING.writeObject(output, it) } + + /* Write data. */ + for ((i, v) in value.values().withIndex()) { + if (v != null) { + this.serializers[i].write(output, v) + } } - Types.Uuid -> UuidValue(buffer.long, buffer.long) - is Types.BooleanVector -> TODO() - is Types.Complex32Vector -> Complex32VectorValue(FloatArray(2*type.logicalSize) { buffer.float }) - is Types.Complex64Vector -> Complex64VectorValue(DoubleArray(2*type.logicalSize) { buffer.double }) - is Types.DoubleVector -> DoubleVectorValue(DoubleArray(type.logicalSize) { buffer.double }) - is Types.FloatVector -> FloatVectorValue(FloatArray(type.logicalSize) { buffer.float }) - is Types.IntVector -> IntVectorValue(IntArray(type.logicalSize) { buffer.int }) - is Types.LongVector -> LongVectorValue(LongArray(type.logicalSize) { buffer.long }) - is Types.ShortVector -> ShortVectorValue(ShortArray(type.logicalSize) { buffer.short }) - is Types.HalfVector -> FloatVectorValue(FloatArray(type.logicalSize) { Half(buffer.short.toUShort()).toFloat() }) } } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanValueValueSerializer.kt index d135b11e1..c6171f3c8 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.BooleanBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.BooleanValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [BooleanValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object BooleanValueValueSerializer: ValueSerializer { override val type = Types.Boolean - override fun fromEntry(entry: ByteIterable): BooleanValue = BooleanValue(BooleanBinding.entryToBoolean(entry)) - override fun toEntry(value: BooleanValue): ByteIterable = BooleanBinding.booleanToEntry(value.value) + override fun write(output: LightOutputStream, value: BooleanValue) = BooleanBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): BooleanValue = BooleanValue(BooleanBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanVectorValueValueSerializer.kt index b20ea5477..fcf40a68f 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/BooleanVectorValueValueSerializer.kt @@ -1,35 +1,27 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.LongBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.BooleanVectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [BooleanVectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ -class BooleanVectorValueValueSerializer(size: Int): ValueSerializer { +class BooleanVectorValueValueSerializer(val size: Int): ValueSerializer { companion object { - /** - * Initialises a [LongArray] for the given number of booleans to store. - * - * @param logicalSize The logical size of the [BooleanVectorValue]. - * @return The corresponding [LongArray]. - */ - protected fun initWordArrayForSize(logicalSize: Int) = LongArray(wordIndex(logicalSize - 1) + 1) - /** * Converts the given [bitIndex] into a word index. * * @param [bitIndex] The bit index to convert. * @return Corresponding word index. */ - protected fun wordIndex(bitIndex: Int): Int = bitIndex.shr(6) + fun wordIndex(bitIndex: Int): Int = bitIndex.shr(6) } init { @@ -39,28 +31,11 @@ class BooleanVectorValueValueSerializer(size: Int): ValueSerializer = Types.BooleanVector(size) - /** - * Converts the given [ByteIterable] into a [BooleanVectorValue]. - * - * @param entry The [ByteIterable] to convert. - * @return The corresponding [BooleanVectorValue] - */ - private fun internalEntryToValue(entry: ByteIterable): BooleanVectorValue { - val wordArray = Snappy.uncompressLongArray(entry.bytesUnsafe) - return BooleanVectorValue(BooleanArray(this.type.logicalSize) { - (wordArray[wordIndex(it)] and (1L shl it)) != 0L - }) - } + /** Number of words that are stored for this [BooleanVectorValue]. */ + private val numberOfWords = wordIndex(this.type.logicalSize - 1) + 1 - /** - * Serializes the given [BooleanVectorValue] and returns a [ByteIterable]. [BooleanVectorValue] - * are encoded into long arrays for more efficient storage. - * - * @param value The [BooleanVectorValue] to serialize. - * @return The corresponding [ByteIterable] - */ - private fun internalValueToEntry(value: BooleanVectorValue): ByteIterable { - val wordArray = initWordArrayForSize(this.type.logicalSize) + override fun write(output: LightOutputStream, value: BooleanVectorValue) { + val wordArray = LongArray(this.numberOfWords) for ((i, v) in value.data.withIndex()) { val wordIndex = wordIndex(i) if (v) { @@ -69,11 +44,17 @@ class BooleanVectorValueValueSerializer(size: Int): ValueSerializer { override val type = Types.ByteString - override fun fromEntry(entry: ByteIterable): ByteStringValue = ByteStringValue(Snappy.uncompress(entry.bytesUnsafe)) - override fun toEntry(value: ByteStringValue): ByteIterable = ArrayByteIterable(Snappy.compress(value.value)) + override fun read(input: ByteArrayInputStream): ByteStringValue = ByteStringValue(input.readAllBytes()) + override fun write(output: LightOutputStream, value: ByteStringValue) { + output.write(value.value) + } } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ByteValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ByteValueValueSerializer.kt index f07e7b033..cfa4f087a 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ByteValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ByteValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.ByteBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.ByteValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [ByteValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object ByteValueValueSerializer: ValueSerializer { override val type = Types.Byte - override fun fromEntry(entry: ByteIterable): ByteValue = ByteValue(ByteBinding.entryToByte(entry)) - override fun toEntry(value: ByteValue): ByteIterable = ByteBinding.byteToEntry(value.value) + override fun write(output: LightOutputStream, value: ByteValue) = ByteBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): ByteValue = ByteValue(ByteBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32ValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32ValueValueSerializer.kt index c74f75b77..70a7b64d7 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32ValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32ValueValueSerializer.kt @@ -1,6 +1,5 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.SignedFloatBinding import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types @@ -11,19 +10,17 @@ import java.io.ByteArrayInputStream * A [ValueSerializer] for [Complex32Value] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object Complex32ValueValueSerializer: ValueSerializer { override val type = Types.Complex32 - override fun fromEntry(entry: ByteIterable): Complex32Value { - val stream = ByteArrayInputStream(entry.bytesUnsafe) - return Complex32Value(SignedFloatBinding.BINDING.readObject(stream), SignedFloatBinding.BINDING.readObject(stream)) + override fun write(output: LightOutputStream, value: Complex32Value) { + SignedFloatBinding.BINDING.writeObject(output, value.real.value) + SignedFloatBinding.BINDING.writeObject(output, value.real.imaginary) } - override fun toEntry(value: Complex32Value): ByteIterable { - val stream = LightOutputStream(this.type.physicalSize) - SignedFloatBinding.BINDING.writeObject(stream, value.real.value) - SignedFloatBinding.BINDING.writeObject(stream, value.real.imaginary) - return stream.asArrayByteIterable() - } + override fun read(input: ByteArrayInputStream) = Complex32Value( + SignedFloatBinding.BINDING.readObject(input), + SignedFloatBinding.BINDING.readObject(input) + ) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32VectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32VectorValueValueSerializer.kt index 50e22fa30..e8eadd121 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32VectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex32VectorValueValueSerializer.kt @@ -1,27 +1,31 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.ComparableBinding +import jetbrains.exodus.bindings.SignedFloatBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.Complex32VectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ComparableBinding] for Xodus based [Complex32VectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ -class Complex32VectorValueValueSerializer(size: Int): ValueSerializer { +class Complex32VectorValueValueSerializer(val size: Int): ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } override val type: Types = Types.Complex32Vector(size) - override fun fromEntry(entry: ByteIterable): Complex32VectorValue = Complex32VectorValue(Snappy.uncompressFloatArray(entry.bytesUnsafe)) - override fun toEntry(value: Complex32VectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + override fun write(output: LightOutputStream, value: Complex32VectorValue) { + for (v in value.data) { + SignedFloatBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = Complex32VectorValue(FloatArray(2 * this.size) { + SignedFloatBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64ValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64ValueValueSerializer.kt index 68c1d34c2..22942f04b 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64ValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64ValueValueSerializer.kt @@ -1,6 +1,5 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.SignedDoubleBinding import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types @@ -11,19 +10,14 @@ import java.io.ByteArrayInputStream * A [ValueSerializer] for [Complex64Value] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object Complex64ValueValueSerializer: ValueSerializer { override val type = Types.Complex64 - override fun fromEntry(entry: ByteIterable): Complex64Value { - val stream = ByteArrayInputStream(entry.bytesUnsafe) - return Complex64Value(SignedDoubleBinding.BINDING.readObject(stream),SignedDoubleBinding.BINDING.readObject(stream)) + override fun write(output: LightOutputStream, value: Complex64Value) { + SignedDoubleBinding.BINDING.writeObject(output, value.real.value) + SignedDoubleBinding.BINDING.writeObject(output, value.real.imaginary) } - override fun toEntry(value: Complex64Value): ByteIterable { - val stream = LightOutputStream(this.type.physicalSize) - SignedDoubleBinding.BINDING.writeObject(stream, value.real.value) - SignedDoubleBinding.BINDING.writeObject(stream, value.real.imaginary) - return stream.asArrayByteIterable() - } + override fun read(input: ByteArrayInputStream) = Complex64Value(SignedDoubleBinding.BINDING.readObject(input),SignedDoubleBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64VectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64VectorValueValueSerializer.kt index b6bf4bb3a..749e28fc6 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64VectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/Complex64VectorValueValueSerializer.kt @@ -1,28 +1,33 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.ComparableBinding +import jetbrains.exodus.bindings.SignedDoubleBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.Complex64VectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ComparableBinding] for Xodus based [Complex64VectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 20.0 + * @version 3.0.0 */ -class Complex64VectorValueValueSerializer(size: Int): ValueSerializer { +class Complex64VectorValueValueSerializer(val size: Int): ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } override val type: Types = Types.Complex64Vector(size) - override fun fromEntry(entry: ByteIterable): Complex64VectorValue = Complex64VectorValue(Snappy.uncompressDoubleArray(entry.bytesUnsafe)) - override fun toEntry(value: Complex64VectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + + override fun write(output: LightOutputStream, value: Complex64VectorValue) { + for (v in value.data) { + SignedDoubleBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = Complex64VectorValue(DoubleArray(2 * this.size) { + SignedDoubleBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DateValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DateValueValueSerializer.kt index 41f3a3668..3fcd5a6a0 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DateValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DateValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.LongBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.DateValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [DateValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object DateValueValueSerializer: ValueSerializer { override val type = Types.Date - override fun fromEntry(entry: ByteIterable): DateValue = DateValue(LongBinding.entryToLong(entry)) - override fun toEntry(value: DateValue): ByteIterable = LongBinding.longToEntry(value.value) + override fun write(output: LightOutputStream, value: DateValue) = LongBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): DateValue = DateValue(LongBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleValueValueSerializer.kt index 3c32cf5b0..51e40249f 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.SignedDoubleBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.DoubleValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [DoubleValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object DoubleValueValueSerializer: ValueSerializer { override val type = Types.Double - override fun fromEntry(entry: ByteIterable): DoubleValue = DoubleValue(SignedDoubleBinding.entryToDouble(entry)) - override fun toEntry(value: DoubleValue): ByteIterable = SignedDoubleBinding.doubleToEntry(value.value) + override fun write(output: LightOutputStream, value: DoubleValue) = SignedDoubleBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): DoubleValue = DoubleValue(SignedDoubleBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleVectorValueValueSerializer.kt index 8d5fa649b..467ae9b49 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/DoubleVectorValueValueSerializer.kt @@ -1,27 +1,32 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.SignedDoubleBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.DoubleVectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [DoubleVectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ -class DoubleVectorValueValueSerializer(size: Int): ValueSerializer { +class DoubleVectorValueValueSerializer(val size: Int): ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } override val type: Types = Types.DoubleVector(size) - override fun fromEntry(entry: ByteIterable): DoubleVectorValue = DoubleVectorValue(Snappy.uncompressDoubleArray(entry.bytesUnsafe)) - override fun toEntry(value: DoubleVectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + + override fun write(output: LightOutputStream, value: DoubleVectorValue) { + for (v in value.data) { + SignedDoubleBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = DoubleVectorValue(DoubleArray(this.size) { + SignedDoubleBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatValueValueSerializer.kt index cc1c9d55d..ea10c867d 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.SignedFloatBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [FloatValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object FloatValueValueSerializer: ValueSerializer { override val type = Types.Float - override fun fromEntry(entry: ByteIterable): FloatValue = FloatValue(SignedFloatBinding.entryToFloat(entry)) - override fun toEntry(value: FloatValue): ByteIterable = SignedFloatBinding.floatToEntry(value.value) + override fun write(output: LightOutputStream, value: FloatValue) = SignedFloatBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): FloatValue = FloatValue(SignedFloatBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatVectorValueValueSerializer.kt index 4bcccda36..071320dc2 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/FloatVectorValueValueSerializer.kt @@ -1,27 +1,32 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.SignedFloatBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatVectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [FloatVectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ -class FloatVectorValueValueSerializer(size: Int): ValueSerializer { +class FloatVectorValueValueSerializer(val size: Int): ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } override val type: Types = Types.FloatVector(size) - override fun fromEntry(entry: ByteIterable): FloatVectorValue = FloatVectorValue(Snappy.uncompressFloatArray(entry.bytesUnsafe)) - override fun toEntry(value: FloatVectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + + override fun write(output: LightOutputStream, value: FloatVectorValue) { + for (v in value.data) { + SignedFloatBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = FloatVectorValue(FloatArray(this.size) { + SignedFloatBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt index 32ad8cd6d..45f100d14 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/HalfVectorValueValueSerializer.kt @@ -2,32 +2,35 @@ package org.vitrivr.cottontail.storage.serializers.values import jetbrains.exodus.ArrayByteIterable import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.ShortBinding +import jetbrains.exodus.bindings.SignedFloatBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatVectorValue import org.vitrivr.cottontail.core.values.HalfVectorValue -import org.vitrivr.cottontail.utilities.math.Half +import org.vitrivr.cottontail.math.Half import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream import java.nio.ByteBuffer /** * A [ValueSerializer] for [FloatVectorValue] serialization and deserialization with 16 bit precision. */ -class HalfVectorValueValueSerializer(size: Int) : ValueSerializer { +class HalfVectorValueValueSerializer(val size: Int) : ValueSerializer { init { require(size > 0) { "Cannot initialize vector value binding with size value of $size." } } override val type: Types = Types.HalfVector(size) - override fun fromEntry(entry: ByteIterable): HalfVectorValue { - val buffer = ByteBuffer.wrap(Snappy.uncompress(entry.bytesUnsafe)).asShortBuffer() - val floats = FloatArray(buffer.remaining()) { Half(buffer[it].toUShort()).toFloat() } - return HalfVectorValue(floats) - } - override fun toEntry(value: HalfVectorValue): ByteIterable { - val halfs = ShortArray(value.data.size) { Half(value.data[it]).toShort() } - val compressed = Snappy.compress(halfs) - return ArrayByteIterable(compressed, compressed.size) + override fun read(input: ByteArrayInputStream) = HalfVectorValue(FloatArray(this.size) { + Half(ShortBinding.BINDING.readObject(input).toUShort()).toFloat() + }) + + override fun write(output: LightOutputStream, value: HalfVectorValue) { + for (v in value.data) { + SignedFloatBinding.BINDING.writeObject(output, Half(v).toShort()) + } } } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntValueValueSerializer.kt index 5c2edf28e..b1ad09a2f 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.IntegerBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.IntValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [IntValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object IntValueValueSerializer: ValueSerializer { override val type = Types.Int - override fun fromEntry(entry: ByteIterable): IntValue = IntValue(IntegerBinding.entryToInt(entry)) - override fun toEntry(value: IntValue): ByteIterable = IntegerBinding.intToEntry(value.value) + override fun write(output: LightOutputStream, value: IntValue) = IntegerBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): IntValue = IntValue(IntegerBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntVectorValueValueSerializer.kt index 9e384da56..6e7252e5a 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/IntVectorValueValueSerializer.kt @@ -1,16 +1,16 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.IntegerBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.IntVectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [IntVectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ class IntVectorValueValueSerializer(val size: Int): ValueSerializer { @@ -19,9 +19,14 @@ class IntVectorValueValueSerializer(val size: Int): ValueSerializer = Types.IntVector(this.size) - override fun fromEntry(entry: ByteIterable): IntVectorValue = IntVectorValue(Snappy.uncompressIntArray(entry.bytesUnsafe)) - override fun toEntry(value: IntVectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + + override fun write(output: LightOutputStream, value: IntVectorValue) { + for (v in value.data) { + IntegerBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = IntVectorValue(IntArray(this.size) { + IntegerBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongValueValueSerializer.kt index eb93d5978..a0692af28 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.LongBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.LongValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [LongValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object LongValueValueSerializer: ValueSerializer { override val type = Types.Long - override fun fromEntry(entry: ByteIterable): LongValue = LongValue(LongBinding.entryToLong(entry)) - override fun toEntry(value: LongValue): ByteIterable = LongBinding.longToEntry(value.value) + override fun write(output: LightOutputStream, value: LongValue) = LongBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): LongValue = LongValue(LongBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongVectorValueValueSerializer.kt index b282e9648..9d1b3159b 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/LongVectorValueValueSerializer.kt @@ -1,16 +1,16 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.LongBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.LongVectorValue -import org.xerial.snappy.Snappy +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [LongVectorValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ class LongVectorValueValueSerializer(val size: Int): ValueSerializer { init { @@ -19,10 +19,13 @@ class LongVectorValueValueSerializer(val size: Int): ValueSerializer = Types.LongVector(this.size) - override fun fromEntry(entry: ByteIterable): LongVectorValue = LongVectorValue(Snappy.uncompressLongArray(entry.bytesUnsafe)) - - override fun toEntry(value: LongVectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + override fun write(output: LightOutputStream, value: LongVectorValue) { + for (v in value.data) { + LongBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream) = LongVectorValue(LongArray(this.size) { + LongBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortValueValueSerializer.kt index 92d2b8c24..a36eefc6a 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortValueValueSerializer.kt @@ -1,18 +1,19 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.ShortBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.ShortValue +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [ShortValue] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object ShortValueValueSerializer: ValueSerializer { override val type = Types.Short - override fun fromEntry(entry: ByteIterable): ShortValue = ShortValue(ShortBinding.entryToShort(entry)) - override fun toEntry(value: ShortValue): ByteIterable = ShortBinding.shortToEntry(value.value) + override fun write(output: LightOutputStream, value: ShortValue) = ShortBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): ShortValue = ShortValue(ShortBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortVectorValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortVectorValueValueSerializer.kt index 01fdc8199..2271bff0f 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortVectorValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ShortVectorValueValueSerializer.kt @@ -1,16 +1,16 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ArrayByteIterable -import jetbrains.exodus.ByteIterable +import jetbrains.exodus.bindings.ShortBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types -import org.vitrivr.cottontail.core.values.LongVectorValue import org.vitrivr.cottontail.core.values.ShortVectorValue -import org.xerial.snappy.Snappy -import java.nio.ByteBuffer +import java.io.ByteArrayInputStream /** * A [ValueSerializer] for [ShortVectorValue] serialization and deserialization. * + * @author Ralph Gasser + * @version 3.0.0 */ class ShortVectorValueValueSerializer(val size: Int): ValueSerializer { init { @@ -18,11 +18,13 @@ class ShortVectorValueValueSerializer(val size: Int): ValueSerializer = Types.ShortVector(this.size) - - override fun fromEntry(entry: ByteIterable): ShortVectorValue = ShortVectorValue(ByteBuffer.wrap(Snappy.uncompress(entry.bytesUnsafe))) - - override fun toEntry(value: ShortVectorValue): ByteIterable { - val compressed = Snappy.compress(value.data) - return ArrayByteIterable(compressed, compressed.size) + override fun write(output: LightOutputStream, value: ShortVectorValue) { + for (v in value.data) { + ShortBinding.BINDING.writeObject(output, v) + } } + + override fun read(input: ByteArrayInputStream): ShortVectorValue = ShortVectorValue(ShortArray(this.size) { + ShortBinding.BINDING.readObject(input) + }) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/StringValueValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/StringValueValueSerializer.kt index 4290ac81e..076906a3c 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/StringValueValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/StringValueValueSerializer.kt @@ -1,19 +1,20 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.ComparableBinding import jetbrains.exodus.bindings.StringBinding +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.StringValue +import java.io.ByteArrayInputStream /** * A [ComparableBinding] for Xodus based [StringBinding] serialization and deserialization. * * @author Ralph Gasser - * @version 2.0.0 + * @version 3.0.0 */ object StringValueValueSerializer: ValueSerializer { override val type = Types.String - override fun fromEntry(entry: ByteIterable): StringValue = StringValue(StringBinding.entryToString(entry)) - override fun toEntry(value: StringValue): ByteIterable = StringBinding.stringToEntry(value.value) + override fun write(output: LightOutputStream, value: StringValue) = StringBinding.BINDING.writeObject(output, value.value) + override fun read(input: ByteArrayInputStream): StringValue = StringValue(StringBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/UuidValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/UuidValueSerializer.kt index 2f0008eda..32d9b74b1 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/UuidValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/UuidValueSerializer.kt @@ -1,6 +1,5 @@ package org.vitrivr.cottontail.storage.serializers.values -import jetbrains.exodus.ByteIterable import jetbrains.exodus.bindings.LongBinding import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types @@ -11,19 +10,14 @@ import java.io.ByteArrayInputStream * A [ValueSerializer] for [UuidValue]s. * * @author Ralph Gasser - * @version 1.0.0 + * @version 3.0.0 */ object UuidValueSerializer: ValueSerializer { override val type = Types.Uuid - override fun fromEntry(entry: ByteIterable): UuidValue { - val stream = ByteArrayInputStream(entry.bytesUnsafe) - return UuidValue(LongBinding.BINDING.readObject(stream), LongBinding.BINDING.readObject(stream)) + override fun write(output: LightOutputStream, value: UuidValue) { + LongBinding.BINDING.writeObject(output, value.value.mostSignificantBits) + LongBinding.BINDING.writeObject(output, value.value.leastSignificantBits) } - override fun toEntry(value: UuidValue): ByteIterable { - val stream = LightOutputStream(this.type.physicalSize) - LongBinding.BINDING.writeObject(stream, value.mostSignificantBits) - LongBinding.BINDING.writeObject(stream, value.leastSignificantBits) - return stream.asArrayByteIterable() - } + override fun read(input: ByteArrayInputStream): UuidValue = UuidValue(LongBinding.BINDING.readObject(input), LongBinding.BINDING.readObject(input)) } \ No newline at end of file diff --git a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ValueSerializer.kt b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ValueSerializer.kt index 57b3176fb..100dc2f08 100644 --- a/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ValueSerializer.kt +++ b/cottontaildb-dbms/src/main/kotlin/org/vitrivr/cottontail/storage/serializers/values/ValueSerializer.kt @@ -1,14 +1,17 @@ package org.vitrivr.cottontail.storage.serializers.values import jetbrains.exodus.ByteIterable +import jetbrains.exodus.util.ByteArraySizedInputStream +import jetbrains.exodus.util.LightOutputStream import org.vitrivr.cottontail.core.types.Types +import java.io.ByteArrayInputStream import org.vitrivr.cottontail.core.types.Value /** * A serializer for Xodus based [Value] serialization and deserialization. * * @author Ralph Gasser - * @version 2.1.0 + * @version 3.0.0 */ sealed interface ValueSerializer { @@ -21,7 +24,7 @@ sealed interface ValueSerializer { * @param entry The [ByteIterable] to convert. * @return The resulting [Value]. */ - fun fromEntry(entry: ByteIterable): T? + fun fromEntry(entry: ByteIterable): T? = this.read(ByteArraySizedInputStream(entry.bytesUnsafe, 0, entry.length)) /** * Converts a [Value] to a [ByteIterable]. @@ -29,5 +32,25 @@ sealed interface ValueSerializer { * @param value The [Value] to convert. * @return The resulting [ByteIterable]. */ - fun toEntry(value: T): ByteIterable + fun toEntry(value: T): ByteIterable { + val output = LightOutputStream() + this.write(output, value) + return output.asArrayByteIterable() + } + + /** + * Reads a [Value] from a [ByteArrayInputStream]. + * + * @param input The [ByteArrayInputStream] to read. + * @return The resulting [Value]. + */ + fun read(input: ByteArrayInputStream): T + + /** + * Writes a [Value] to a [LightOutputStream]. + * + * @param output The [LightOutputStream] to read. + * @return The resulting [Value]. + */ + fun write(output: LightOutputStream, value: T) } \ No newline at end of file