Array fields support (#81)

Fixes #77
milvus-io · Apr 22, 2024 · b1bd801 · b1bd801
1 parent de2608d
commit b1bd801
Show file tree

Hide file tree

Showing 8 changed files with 428 additions and 33 deletions.
diff --git a/Milvus.Client.Tests/FieldTests.cs b/Milvus.Client.Tests/FieldTests.cs
@@ -67,4 +67,141 @@ public void CreateFloatVectorTest()
         Assert.Equal(2, field.RowCount);
         Assert.Equal(2, field.Data.Count);
     }
+
+    [Fact]
+    public void CreateInt8ArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new sbyte[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Int8, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateInt16ArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new short[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Int16, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateInt32ArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new int[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Int32, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateInt64ArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new long[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Int64, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateBoolArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new bool[][]
+            {
+                [true, false],
+                [false, false],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Bool, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateFloatArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new float[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Float, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    [Fact]
+    public void CreateDoubleArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new double[][]
+            {
+                [1, 2],
+                [3, 4],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.Double, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
+
+    //TODO: differentiate VarChar and String somehow
+    [Fact]
+    public void CreateVarCharArrayTest()
+    {
+        var field = FieldData.CreateArray(
+            "vector",
+            new string[][]
+            {
+                ["3d4d387208e04a9abe77be65e2b7c7b3", "a5502ddb557047968a70ff69720d2dd2"],
+                ["4c246789a91f4b15aa3b26799df61457", "00a23e95823b4f14854ceed5f7059953"],
+            });
+
+        Assert.Equal(MilvusDataType.Array, field.DataType);
+        Assert.Equal(MilvusDataType.VarChar, field.ElementType);
+        Assert.Equal(2, field.RowCount);
+        Assert.Equal(2, field.Data.Count);
+    }
 }
diff --git a/Milvus.Client.sln.DotSettings b/Milvus.Client.sln.DotSettings
@@ -1,9 +1,12 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/BLANK_LINES_AROUND_MULTILINE_CASE_SECTION/@EntryValue">1</s:Int64>
+	<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/SPACE_WITHIN_SINGLE_LINE_ARRAY_INITIALIZER_BRACES/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Jaccard/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Milvus/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pymilvus/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizes/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizing/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=sbytes/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Tanimoto/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=topk/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=upserted/@EntryIndexedValue">True</s:Boolean>

diff --git a/Milvus.Client/ArrayFieldData.cs b/Milvus.Client/ArrayFieldData.cs
@@ -0,0 +1,145 @@
+namespace Milvus.Client;
+
+/// <summary>
+/// Binary Field
+/// </summary>
+public sealed class ArrayFieldData<TElementData> : FieldData<IReadOnlyList<TElementData>>
+{
+    /// <summary>
+    /// Construct an array field
+    /// </summary>
+    /// <param name="fieldName"></param>
+    /// <param name="data"></param>
+    /// <param name="isDynamic"></param>
+    public ArrayFieldData(string fieldName, IReadOnlyList<IReadOnlyList<TElementData>> data, bool isDynamic)
+        : base(fieldName, data, MilvusDataType.Array, isDynamic)
+    {
+        ElementType = EnsureDataType<TElementData>();
+    }
+
+    /// <summary>
+    /// Array element type
+    /// </summary>
+    public MilvusDataType ElementType { get; }
+
+    /// <inheritdoc />
+    internal override Grpc.FieldData ToGrpcFieldData()
+    {
+        Check();
+
+        Grpc.FieldData fieldData = new()
+        {
+            Type = Grpc.DataType.Array,
+            IsDynamic = IsDynamic
+        };
+
+        if (FieldName is not null)
+        {
+            fieldData.FieldName = FieldName;
+        }
+
+        var arrayArray = new ArrayArray
+        {
+            ElementType = (DataType) ElementType,
+        };
+
+        fieldData.Scalars = new ScalarField
+        {
+            ArrayData = arrayArray
+        };
+
+        foreach (var array in Data)
+        {
+            switch (ElementType)
+            {
+                case MilvusDataType.Bool:
+                    BoolArray boolData = new();
+                    boolData.Data.AddRange(array as IEnumerable<bool>);
+                    arrayArray.Data.Add(new ScalarField { BoolData = boolData, });
+                    break;
+
+                case MilvusDataType.Int8:
+                    IntArray int8Data = new();
+                    var sbytes = array as IEnumerable<sbyte> ?? Enumerable.Empty<sbyte>();
+                    int8Data.Data.AddRange(sbytes.Select(x => (int) x));
+                    arrayArray.Data.Add(new ScalarField { IntData = int8Data, });
+                    break;
+
+                case MilvusDataType.Int16:
+                    IntArray int16Data = new();
+                    var shorts = array as IEnumerable<short> ?? Enumerable.Empty<short>();
+                    int16Data.Data.AddRange(shorts.Select(x => (int) x));
+                    arrayArray.Data.Add(new ScalarField { IntData = int16Data, });
+                    break;
+
+                case MilvusDataType.Int32:
+                    IntArray int32Data = new();
+                    int32Data.Data.AddRange(array as IEnumerable<int>);
+                    arrayArray.Data.Add(new ScalarField { IntData = int32Data, });
+                    break;
+
+                case MilvusDataType.Int64:
+                    LongArray int64Data = new();
+                    int64Data.Data.AddRange(array as IEnumerable<long>);
+                    arrayArray.Data.Add(new ScalarField { LongData = int64Data, });
+                    break;
+
+                case MilvusDataType.Float:
+                    FloatArray floatData = new();
+                    floatData.Data.AddRange(array as IEnumerable<float>);
+                    arrayArray.Data.Add(new ScalarField { FloatData = floatData, });
+                    break;
+
+                case MilvusDataType.Double:
+                    DoubleArray doubleData = new();
+                    doubleData.Data.AddRange(array as IEnumerable<double>);
+                    arrayArray.Data.Add(new ScalarField { DoubleData = doubleData, });
+                    break;
+
+                case MilvusDataType.String:
+                    StringArray stringData = new();
+                    stringData.Data.AddRange(array as IEnumerable<string>);
+                    arrayArray.Data.Add(new ScalarField { StringData = stringData, });
+                    break;
+
+                case MilvusDataType.VarChar:
+                    StringArray varcharData = new();
+                    varcharData.Data.AddRange(array as IEnumerable<string>);
+                    arrayArray.Data.Add(new ScalarField { StringData = varcharData, });
+                    break;
+
+                case MilvusDataType.Json:
+                    JSONArray jsonData = new();
+                    var enumerable = array as IEnumerable<string> ?? Enumerable.Empty<string>();
+                    jsonData.Data.AddRange(enumerable.Select(ByteString.CopyFromUtf8));
+                    arrayArray.Data.Add(new ScalarField { JsonData = jsonData, });
+                    break;
+
+                case MilvusDataType.None:
+                    throw new MilvusException($"ElementType Error:{DataType}");
+
+                default:
+                    throw new MilvusException($"ElementType Error:{DataType}, not supported");
+            }
+        }
+
+        return fieldData;
+    }
+
+    internal override object GetValueAsObject(int index)
+        => ElementType switch
+        {
+            MilvusDataType.Bool => ((IReadOnlyList<IEnumerable<bool>>) Data)[index],
+            MilvusDataType.Int8 => ((IReadOnlyList<IEnumerable<sbyte>>) Data)[index],
+            MilvusDataType.Int16 => ((IReadOnlyList<IEnumerable<short>>) Data)[index],
+            MilvusDataType.Int32 => ((IReadOnlyList<IEnumerable<int>>) Data)[index],
+            MilvusDataType.Int64 => ((IReadOnlyList<IEnumerable<long>>) Data)[index],
+            MilvusDataType.Float => ((IReadOnlyList<IEnumerable<float>>) Data)[index],
+            MilvusDataType.Double => ((IReadOnlyList<IEnumerable<double>>) Data)[index],
+            MilvusDataType.String => ((IReadOnlyList<IEnumerable<string>>) Data)[index],
+            MilvusDataType.VarChar => ((IReadOnlyList<IEnumerable<string>>) Data)[index],
+
+            MilvusDataType.None => throw new MilvusException($"DataType Error:{DataType}"),
+            _ => throw new MilvusException($"DataType Error:{DataType}, not supported")
+        };
+}
diff --git a/Milvus.Client/Constants.cs b/Milvus.Client/Constants.cs
@@ -70,6 +70,11 @@ internal static class Constants
     /// </summary>
     internal const string FailedReason = "failed_reason";
 
+    /// <summary>
+    /// Key name.
+    /// </summary>
+    internal const string MaxCapacity = "max_capacity";
+
     /// <summary>
     /// Files.
     /// </summary>