diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java index cd77ebd9a..eeaef9847 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.mapper; +import org.opensearch.Version; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; @@ -62,4 +63,12 @@ default QuantizationConfig getQuantizationConfig() { * @return the dimension of the index; for model based indices, it will be null */ int getDimension(); + + /** + * Returns index created Version + * @return Version + */ + default Version getIndexCreatedVersion() { + return Version.CURRENT; + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 7990fdcab..4ceb9b4b2 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -17,6 +17,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnFloatVectorField; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.knn.index.KNNVectorSimilarityFunction; import org.opensearch.knn.index.VectorDataType; @@ -73,6 +74,11 @@ public Mode getMode() { public CompressionLevel getCompressionLevel() { return knnMethodConfigContext.getCompressionLevel(); } + + @Override + public Version getIndexCreatedVersion() { + return knnMethodConfigContext.getVersionCreated(); + } } ); diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index bf5bc2b51..755439ce6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -8,6 +8,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.VectorEncoding; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.SpaceType; @@ -86,6 +87,11 @@ public CompressionLevel getCompressionLevel() { public QuantizationConfig getQuantizationConfig() { return quantizationConfig; } + + @Override + public Version getIndexCreatedVersion() { + return knnMethodConfigContext.getVersionCreated(); + } } ); return new MethodFieldMapper( diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 013cb0c53..cbc7520cf 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -107,6 +107,11 @@ public QuantizationConfig getQuantizationConfig() { return quantizationConfig; } + @Override + public Version getIndexCreatedVersion() { + return indexCreatedVersion; + } + // ModelMetadata relies on cluster state which may not be available during field mapper creation. Thus, // we lazily initialize it. private void initFromModelMetadata() { diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 9744796c6..b613efab2 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -8,6 +8,7 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; +import org.opensearch.Version; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -69,7 +70,7 @@ public KNNFieldSpace( ) { KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType(fieldType, spaceName, supportingVectorDataTypes); this.processedQuery = getProcessedQuery(query, knnVectorFieldType); - this.scoringMethod = getScoringMethod(this.processedQuery); + this.scoringMethod = getScoringMethod(this.processedQuery, knnVectorFieldType.getKnnMappingConfig().getIndexCreatedVersion()); } public ScoreScript getScoreScript( @@ -122,6 +123,10 @@ protected float[] getProcessedQuery(final Object query, final KNNVectorFieldType protected abstract BiFunction getScoringMethod(final float[] processedQuery); + protected BiFunction getScoringMethod(final float[] processedQuery, Version indexCreatedVersion) { + return getScoringMethod(processedQuery); + } + } class L2 extends KNNFieldSpace { @@ -141,15 +146,30 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) { } @Override - protected BiFunction getScoringMethod(final float[] processedQuery) { + protected BiFunction getScoringMethod(float[] processedQuery) { + return getScoringMethod(processedQuery, Version.CURRENT); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery, Version indexCreatedVersion) { SpaceType.COSINESIMIL.validateVector(processedQuery); float qVectorSquaredMagnitude = getVectorMagnitudeSquared(processedQuery); - // To be consistent, we will be using same formula used by lucene as mentioned below - // https://github.com/apache/lucene/blob/0494c824e0ac8049b757582f60d085932a890800/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java#L73 - return (float[] q, float[] v) -> Math.max( - (1.0F + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude)) / 2.0F, - 0.0F - ); + if (indexCreatedVersion.onOrAfter(Version.V_2_19_0)) { + // To be consistent, we will be using same formula used by lucene as mentioned below + // https://github.com/apache/lucene/blob/0494c824e0ac8049b757582f60d085932a890800/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java#L73 + // for indices that are created on or after 2.19.0 + // + // OS Score = ( 2 − cosineSimil) / 2 + // However cosineSimil = 1 - cos θ, after applying this to above formula, + // OS Score = ( 2 − ( 1 − cos θ ) ) / 2 + // which simplifies to + // OS Score = ( 1 + cos θ ) / 2 + return (float[] q, float[] v) -> Math.max( + ((1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude)) / 2.0F), + 0 + ); + } + return (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); } }