add version check

Signed-off-by: Vijayan Balasubramanian <[email protected]>
opensearch-project · Jan 2, 2025 · 49668d0 · 49668d0
1 parent 87d0018
commit 49668d0
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 8 deletions.
diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java
@@ -5,6 +5,7 @@
 
 package org.opensearch.knn.index.mapper;
 
+import org.opensearch.Version;
 import org.opensearch.knn.index.engine.KNNMethodContext;
 import org.opensearch.knn.index.engine.qframe.QuantizationConfig;
 
@@ -62,4 +63,12 @@ default QuantizationConfig getQuantizationConfig() {
      * @return the dimension of the index; for model based indices, it will be null
      */
     int getDimension();
+
+    /**
+     * Returns index created Version
+     * @return Version
+     */
+    default Version getIndexCreatedVersion() {
+        return Version.CURRENT;
+    }
 }
diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.KnnByteVectorField;
 import org.apache.lucene.document.KnnFloatVectorField;
+import org.opensearch.Version;
 import org.opensearch.common.Explicit;
 import org.opensearch.knn.index.KNNVectorSimilarityFunction;
 import org.opensearch.knn.index.VectorDataType;
@@ -73,6 +74,11 @@ public Mode getMode() {
                 public CompressionLevel getCompressionLevel() {
                     return knnMethodConfigContext.getCompressionLevel();
                 }
+
+                @Override
+                public Version getIndexCreatedVersion() {
+                    return knnMethodConfigContext.getVersionCreated();
+                }
             }
         );
 

diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java
@@ -8,6 +8,7 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.VectorEncoding;
+import org.opensearch.Version;
 import org.opensearch.common.Explicit;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.knn.index.SpaceType;
@@ -86,6 +87,11 @@ public CompressionLevel getCompressionLevel() {
                 public QuantizationConfig getQuantizationConfig() {
                     return quantizationConfig;
                 }
+
+                @Override
+                public Version getIndexCreatedVersion() {
+                    return knnMethodConfigContext.getVersionCreated();
+                }
             }
         );
         return new MethodFieldMapper(

diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java
@@ -107,6 +107,11 @@ public QuantizationConfig getQuantizationConfig() {
                 return quantizationConfig;
             }
 
+            @Override
+            public Version getIndexCreatedVersion() {
+                return indexCreatedVersion;
+            }
+
             // ModelMetadata relies on cluster state which may not be available during field mapper creation. Thus,
             // we lazily initialize it.
             private void initFromModelMetadata() {

diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java
@@ -8,6 +8,7 @@
 import lombok.Getter;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
+import org.opensearch.Version;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.VectorDataType;
@@ -69,7 +70,7 @@ public KNNFieldSpace(
         ) {
             KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType(fieldType, spaceName, supportingVectorDataTypes);
             this.processedQuery = getProcessedQuery(query, knnVectorFieldType);
-            this.scoringMethod = getScoringMethod(this.processedQuery);
+            this.scoringMethod = getScoringMethod(this.processedQuery, knnVectorFieldType.getKnnMappingConfig().getIndexCreatedVersion());
         }
 
         public ScoreScript getScoreScript(
@@ -122,6 +123,10 @@ protected float[] getProcessedQuery(final Object query, final KNNVectorFieldType
 
         protected abstract BiFunction<float[], float[], Float> getScoringMethod(final float[] processedQuery);
 
+        protected BiFunction<float[], float[], Float> getScoringMethod(final float[] processedQuery, Version indexCreatedVersion) {
+            return getScoringMethod(processedQuery);
+        }
+
     }
 
     class L2 extends KNNFieldSpace {
@@ -141,15 +146,30 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) {
         }
 
         @Override
-        protected BiFunction<float[], float[], Float> getScoringMethod(final float[] processedQuery) {
+        protected BiFunction<float[], float[], Float> getScoringMethod(float[] processedQuery) {
+            return getScoringMethod(processedQuery, Version.CURRENT);
+        }
+
+        @Override
+        protected BiFunction<float[], float[], Float> getScoringMethod(final float[] processedQuery, Version indexCreatedVersion) {
             SpaceType.COSINESIMIL.validateVector(processedQuery);
             float qVectorSquaredMagnitude = getVectorMagnitudeSquared(processedQuery);
-            // To be consistent, we will be using same formula used by lucene as mentioned below
-            // https://github.com/apache/lucene/blob/0494c824e0ac8049b757582f60d085932a890800/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java#L73
-            return (float[] q, float[] v) -> Math.max(
-                (1.0F + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude)) / 2.0F,
-                0.0F
-            );
+            if (indexCreatedVersion.onOrAfter(Version.V_2_19_0)) {
+                // To be consistent, we will be using same formula used by lucene as mentioned below
+                // https://github.com/apache/lucene/blob/0494c824e0ac8049b757582f60d085932a890800/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java#L73
+                // for indices that are created on or after 2.19.0
+                //
+                // OS Score = ( 2 − cosineSimil) / 2
+                // However cosineSimil = 1 - cos θ, after applying this to above formula,
+                // OS Score = ( 2 − ( 1 − cos θ ) ) / 2
+                // which simplifies to
+                // OS Score = ( 1 + cos θ ) / 2
+                return (float[] q, float[] v) -> Math.max(
+                    (1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude) / 2.0F),
+                    0
+                );
+            }
+            return (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude);
         }
     }