From 58c1854297959e056e46d7d7ba43825d3377165b Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 13:03:34 -0800 Subject: [PATCH 1/8] Bump lucene version to 9.9.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 50d161860..c33f80c99 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ def _artifactId = 'server' //This is for https://github.com/gradle/gradle/issues/11308 System.setProperty("org.gradle.internal.publish.checksums.insecure", "True") -def luceneVersion = '9.8.0' +def luceneVersion = '9.9.0' project.ext.slf4jVersion = '2.0.0-alpha1' project.ext.grpcVersion = '1.46.0' project.ext.lz4Version = '1.7.0' From 9213fc3929ffeeaec0e525913ae1a1f20b6a60f7 Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 13:05:49 -0800 Subject: [PATCH 2/8] Replaced all Lucene95 codecs with Lucene99 --- .../nrtsearch/server/luceneserver/ServerCodec.java | 4 ++-- .../server/luceneserver/field/VectorFieldDef.java | 12 ++++++------ .../suggest/CompletionInfixSuggester.java | 4 ++-- .../luceneserver/field/VectorFieldDefTest.java | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java index ed2e5fe8f..70f8605ed 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java @@ -23,10 +23,10 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; /** Implements per-index {@link Codec}. */ -public class ServerCodec extends Lucene95Codec { +public class ServerCodec extends Lucene99Codec { public static final String DEFAULT_POSTINGS_FORMAT = "Lucene90"; public static final String DEFAULT_DOC_VALUES_FORMAT = "Lucene90"; diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java index 996a1848c..2e369d1ee 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.KnnFloatVectorField; @@ -77,12 +77,12 @@ private static KnnVectorsFormat createVectorsFormat(VectorIndexingOptions vector int m = vectorIndexingOptions.getHnswM() > 0 ? vectorIndexingOptions.getHnswM() - : Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN; + : Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = vectorIndexingOptions.getHnswEfConstruction() > 0 ? vectorIndexingOptions.getHnswEfConstruction() - : Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - return new Lucene95HnswVectorsFormat(m, efConstruction); + : Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + return new Lucene99HnswVectorsFormat(m, efConstruction); } /** @@ -134,10 +134,10 @@ protected void validateRequest(Field requestField) { } if (requestField.getSearch()) { - if (requestField.getVectorDimensions() > Lucene95HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS) { + if (requestField.getVectorDimensions() > Lucene99HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS) { throw new IllegalArgumentException( "Vector dimension must be <= " - + Lucene95HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS + + Lucene99HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS + " for search"); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java index cec15c1bb..b56a4adbc 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java @@ -24,7 +24,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; @@ -249,7 +249,7 @@ protected IndexWriterConfig getIndexWriterConfig( Analyzer indexAnalyzer, IndexWriterConfig.OpenMode mode) { IndexWriterConfig iwc = super.getIndexWriterConfig(indexAnalyzer, mode); Codec filterCodec = - new Lucene95Codec() { + new Lucene99Codec() { final PostingsFormat fstPostingsFormat = new Completion90PostingsFormat(); @Override diff --git a/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java index 3ab07d204..db9aa7a0f 100644 --- a/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java @@ -695,7 +695,7 @@ public void testVectorFormat_set_m() { KnnVectorsFormat format = vectorFieldDef.getVectorsFormat(); assertNotNull(format); assertEquals( - "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn=5, beamWidth=100)", + "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=5, beamWidth=100)", format.toString()); } @@ -718,7 +718,7 @@ public void testVectorFormat_set_ef_construction() { KnnVectorsFormat format = vectorFieldDef.getVectorsFormat(); assertNotNull(format); assertEquals( - "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn=16, beamWidth=50)", + "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=16, beamWidth=50)", format.toString()); } From 4b397ae33647c11b36354501251fdd17c9dd07ae Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 13:19:38 -0800 Subject: [PATCH 3/8] Replaced rewrite(IndexReader) with rewrite(IndexSearcher) --- .../search/query/BlendedTermQuery.java | 12 ++++++------ .../search/query/MatchPhrasePrefixQuery.java | 7 ++++--- .../search/query/multifunction/FilterFunction.java | 14 +++++++------- .../multifunction/MultiFunctionScoreQuery.java | 9 ++++----- .../query/multifunction/ScriptFilterFunction.java | 5 +---- .../query/multifunction/WeightFilterFunction.java | 5 +---- 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java index 44511b15f..9b2269270 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.TermQuery; @@ -78,21 +79,20 @@ public BlendedTermQuery(Term[] terms, float[] boosts) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } - IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; for (int i = 0; i < terms.length; i++) { - ctx[i] = TermStates.build(context, terms[i], true); + ctx[i] = TermStates.build(indexSearcher, terms[i], true); docFreqs[i] = ctx[i].docFreq(); } - final int maxDoc = reader.maxDoc(); - blend(ctx, maxDoc, reader); + final int maxDoc = indexSearcher.getIndexReader().maxDoc(); + blend(ctx, maxDoc, indexSearcher.getIndexReader()); return topLevelQuery(terms, ctx, docFreqs, maxDoc); } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java index f3a8cdde0..f574d91c7 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java @@ -42,6 +42,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; @@ -193,8 +194,8 @@ public MatchPhrasePrefixQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } @@ -211,7 +212,7 @@ public Query rewrite(IndexReader reader) throws IOException { int position = positions.get(sizeMinus1); Set terms = new HashSet<>(); for (Term term : suffixTerms) { - getPrefixTerms(terms, term, reader); + getPrefixTerms(terms, term, indexSearcher.getIndexReader()); if (terms.size() > maxExpansions) { break; } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java index b73b9e9a4..a0a6f3115 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; /** @@ -133,31 +134,30 @@ public float getWeight() { /** * Method to rewrite queries with the given {@link IndexReader}. Final to force use of {@link - * #doRewrite(IndexReader, boolean, Query)}. + * #doRewrite(boolean, Query)}. * - * @param reader index reader + * @param indexSearcher index searcher * @return function object with any query rewriting done * @throws IOException */ - public final FilterFunction rewrite(IndexReader reader) throws IOException { + public final FilterFunction rewrite(IndexSearcher indexSearcher) throws IOException { Query rewrittenFilterQuery = null; if (filterQuery != null) { - rewrittenFilterQuery = filterQuery.rewrite(reader); + rewrittenFilterQuery = filterQuery.rewrite(indexSearcher); } - return doRewrite(reader, rewrittenFilterQuery != filterQuery, rewrittenFilterQuery); + return doRewrite(rewrittenFilterQuery != filterQuery, rewrittenFilterQuery); } /** * Rewrite method for all child classes. * - * @param reader index reader * @param filterQueryRewritten if the filter query was modified by rewrite * @param rewrittenFilterQuery final value of rewritten query, may be null if no filter * @return fully rewritten filter function * @throws IOException */ protected abstract FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) + boolean filterQueryRewritten, Query rewrittenFilterQuery) throws IOException; @Override diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java index 934c32847..667472dff 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java @@ -27,7 +27,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -124,16 +123,16 @@ public MultiFunctionScoreQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } - Query rewrittenInner = innerQuery.rewrite(reader); + Query rewrittenInner = innerQuery.rewrite(indexSearcher); boolean needsRewrite = rewrittenInner != innerQuery; FilterFunction[] rewrittenFunctions = new FilterFunction[functions.length]; for (int i = 0; i < functions.length; ++i) { - rewrittenFunctions[i] = functions[i].rewrite(reader); + rewrittenFunctions[i] = functions[i].rewrite(indexSearcher); needsRewrite |= (rewrittenFunctions[i] != functions[i]); } if (needsRewrite) { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java index 809b58fc9..309ebb8a6 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java @@ -21,7 +21,6 @@ import com.yelp.nrtsearch.server.luceneserver.search.query.QueryUtils.SettableDoubleValues; import java.io.IOException; import java.util.Objects; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; @@ -54,9 +53,7 @@ public ScriptFilterFunction( } @Override - protected FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException { + protected FilterFunction doRewrite(boolean filterQueryRewritten, Query rewrittenFilterQuery) { if (filterQueryRewritten) { return new ScriptFilterFunction(rewrittenFilterQuery, getWeight(), script, scriptSource); } else { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java index 2c98768ae..dc9794860 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java @@ -16,7 +16,6 @@ package com.yelp.nrtsearch.server.luceneserver.search.query.multifunction; import java.io.IOException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Query; @@ -37,9 +36,7 @@ public WeightFilterFunction(Query filterQuery, float weight) { } @Override - protected FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException { + protected FilterFunction doRewrite(boolean filterQueryRewritten, Query rewrittenFilterQuery) { if (filterQueryRewritten) { return new WeightFilterFunction(rewrittenFilterQuery, getWeight()); } else { From d8f4666c958a9c235e10fd2a0581ac91538057e4 Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 13:33:52 -0800 Subject: [PATCH 4/8] Added hasBlocks=False to SegmentInfo constructor --- .../server/luceneserver/index/BucketedTieredMergePolicyTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/yelp/nrtsearch/server/luceneserver/index/BucketedTieredMergePolicyTest.java b/src/test/java/com/yelp/nrtsearch/server/luceneserver/index/BucketedTieredMergePolicyTest.java index 8dd9a02ce..d32ff78fe 100644 --- a/src/test/java/com/yelp/nrtsearch/server/luceneserver/index/BucketedTieredMergePolicyTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/luceneserver/index/BucketedTieredMergePolicyTest.java @@ -465,6 +465,7 @@ private SegmentInfos getInfos(List sizes, List deletions) { String.valueOf(id), sizes.get(i), false, + false, null, Collections.emptyMap(), new byte[StringHelper.ID_LENGTH], From 889257e9bc02a90910666fcb70c22689abbcc5e1 Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 15:57:03 -0800 Subject: [PATCH 5/8] Switched to Lucene99 and Completion99PostingsFormat --- .../server/luceneserver/ServerCodec.java | 2 +- .../suggest/CompletionInfixSuggester.java | 4 ++-- ...t.java => Completion99PostingsFormat.java} | 20 +++++++++---------- .../CompletionPostingsFormatUtil.java | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) rename src/main/java/org/apache/lucene/search/suggest/document/{Completion90PostingsFormat.java => Completion99PostingsFormat.java} (72%) diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java index 70f8605ed..56bf368a5 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java @@ -28,7 +28,7 @@ /** Implements per-index {@link Codec}. */ public class ServerCodec extends Lucene99Codec { - public static final String DEFAULT_POSTINGS_FORMAT = "Lucene90"; + public static final String DEFAULT_POSTINGS_FORMAT = "Lucene99"; public static final String DEFAULT_DOC_VALUES_FORMAT = "Lucene90"; private final IndexStateManager stateManager; diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java index b56a4adbc..67dce5742 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.suggest.InputIterator; import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.ContextQuery; import org.apache.lucene.search.suggest.document.ContextSuggestField; @@ -250,7 +250,7 @@ protected IndexWriterConfig getIndexWriterConfig( IndexWriterConfig iwc = super.getIndexWriterConfig(indexAnalyzer, mode); Codec filterCodec = new Lucene99Codec() { - final PostingsFormat fstPostingsFormat = new Completion90PostingsFormat(); + final PostingsFormat fstPostingsFormat = new Completion99PostingsFormat(); @Override public PostingsFormat getPostingsFormatForField(String field) { diff --git a/src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java similarity index 72% rename from src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java rename to src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java index 7da2411e9..485f18abf 100644 --- a/src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java +++ b/src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java @@ -20,32 +20,32 @@ import org.slf4j.LoggerFactory; /** - * Copy of the lucene Completion90PostingsFormat, but allows the FST load mode to be configured. + * Copy of the lucene Completion99PostingsFormat, but allows the FST load mode to be configured. * Since this codec is loaded by class name, it must have the same name as the original and be * present earlier in the class path. */ -public class Completion90PostingsFormat extends CompletionPostingsFormat { - private static final Logger logger = LoggerFactory.getLogger(Completion90PostingsFormat.class); +public class Completion99PostingsFormat extends CompletionPostingsFormat { + private static final Logger logger = LoggerFactory.getLogger(Completion99PostingsFormat.class); /** - * Creates a {@link Completion90PostingsFormat} that will load the completion FST based on the + * Creates a {@link Completion99PostingsFormat} that will load the completion FST based on the * value present in {@link CompletionPostingsFormatUtil}. */ - public Completion90PostingsFormat() { + public Completion99PostingsFormat() { this(CompletionPostingsFormatUtil.getCompletionCodecLoadMode()); } /** - * Creates a {@link Completion90PostingsFormat} that will use the provided fstLoadMode + * Creates a {@link Completion99PostingsFormat} that will use the provided fstLoadMode * to determine if the completion FST should be loaded on or off heap. */ - public Completion90PostingsFormat(FSTLoadMode fstLoadMode) { - super("Completion90", fstLoadMode); - logger.info("Created Completion90PostingsFormat with fstLoadMode: " + fstLoadMode); + public Completion99PostingsFormat(FSTLoadMode fstLoadMode) { + super("Completion99", fstLoadMode); + logger.info("Created Completion99PostingsFormat with fstLoadMode: " + fstLoadMode); } @Override protected PostingsFormat delegatePostingsFormat() { - return PostingsFormat.forName("Lucene90"); + return PostingsFormat.forName("Lucene99"); } } diff --git a/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java b/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java index f42da2a82..d5eea0063 100644 --- a/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java +++ b/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java @@ -24,7 +24,7 @@ public class CompletionPostingsFormatUtil { private CompletionPostingsFormatUtil() {} /** - * Set the FST load mode used by the modified {@link Completion90PostingsFormat}. Must be set + * Set the FST load mode used by the modified {@link Completion99PostingsFormat}. Must be set * before any index data is loaded. * * @param loadMode new FST load mode From 5e217b6c537d4bd20dc38bea3a523a83a5902ba2 Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 15:58:38 -0800 Subject: [PATCH 6/8] Fixed VectorFieldDefTest --- .../server/luceneserver/field/VectorFieldDefTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java index db9aa7a0f..e6a4ac06a 100644 --- a/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java @@ -695,7 +695,7 @@ public void testVectorFormat_set_m() { KnnVectorsFormat format = vectorFieldDef.getVectorsFormat(); assertNotNull(format); assertEquals( - "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=5, beamWidth=100)", + "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=5, beamWidth=100, flatVectorFormat=Lucene99FlatVectorsFormat())", format.toString()); } @@ -718,7 +718,7 @@ public void testVectorFormat_set_ef_construction() { KnnVectorsFormat format = vectorFieldDef.getVectorsFormat(); assertNotNull(format); assertEquals( - "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=16, beamWidth=50)", + "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=16, beamWidth=50, flatVectorFormat=Lucene99FlatVectorsFormat())", format.toString()); } From d8ad3758ef718edde67a7e698076b0d4b16329f4 Mon Sep 17 00:00:00 2001 From: Sarthak Nandi Date: Sat, 2 Mar 2024 16:02:17 -0800 Subject: [PATCH 7/8] Fixed explain test --- .../java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java index d1a80fa4e..540b09f60 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java @@ -214,7 +214,9 @@ public void testExplain() { var explain = hit.getExplain(); var expectedExplain = String.format( - "%d.0 = weight(FunctionScoreQuery(int_field:[0 TO 100], scored by expr(int_score))), result of:\n" + "%d.0 = weight(FunctionScoreQuery(IndexOrDocValuesQuery(" + + "indexQuery=int_field:[0 TO 100], dvQuery=int_field:[0 TO 100]), " + + "scored by expr(int_score))), result of:\n" + " % Date: Sat, 2 Mar 2024 16:07:26 -0800 Subject: [PATCH 8/8] Updated postings format to Completion99 in ContextSuggestFieldDef --- .../server/luceneserver/field/ContextSuggestFieldDef.java | 3 ++- .../search/query/multifunction/FilterFunction.java | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java index b96685e20..823fa507c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java @@ -112,7 +112,8 @@ public Optional getSearchAnalyzer() { return Optional.ofNullable(this.searchAnalyzer); } + @Override public String getPostingsFormat() { - return "Completion90"; + return "Completion99"; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java index a0a6f3115..5a7b6d974 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java @@ -157,8 +157,7 @@ public final FilterFunction rewrite(IndexSearcher indexSearcher) throws IOExcept * @throws IOException */ protected abstract FilterFunction doRewrite( - boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException; + boolean filterQueryRewritten, Query rewrittenFilterQuery) throws IOException; @Override public String toString() {