diff --git a/build.gradle b/build.gradle index 50d161860..c33f80c99 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ def _artifactId = 'server' //This is for https://github.com/gradle/gradle/issues/11308 System.setProperty("org.gradle.internal.publish.checksums.insecure", "True") -def luceneVersion = '9.8.0' +def luceneVersion = '9.9.0' project.ext.slf4jVersion = '2.0.0-alpha1' project.ext.grpcVersion = '1.46.0' project.ext.lz4Version = '1.7.0' diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java index ed2e5fe8f..56bf368a5 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java @@ -23,12 +23,12 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; /** Implements per-index {@link Codec}. */ -public class ServerCodec extends Lucene95Codec { +public class ServerCodec extends Lucene99Codec { - public static final String DEFAULT_POSTINGS_FORMAT = "Lucene90"; + public static final String DEFAULT_POSTINGS_FORMAT = "Lucene99"; public static final String DEFAULT_DOC_VALUES_FORMAT = "Lucene90"; private final IndexStateManager stateManager; diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java index b96685e20..823fa507c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java @@ -112,7 +112,8 @@ public Optional getSearchAnalyzer() { return Optional.ofNullable(this.searchAnalyzer); } + @Override public String getPostingsFormat() { - return "Completion90"; + return "Completion99"; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java index 996a1848c..2e369d1ee 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDef.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.KnnFloatVectorField; @@ -77,12 +77,12 @@ private static KnnVectorsFormat createVectorsFormat(VectorIndexingOptions vector int m = vectorIndexingOptions.getHnswM() > 0 ? vectorIndexingOptions.getHnswM() - : Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN; + : Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = vectorIndexingOptions.getHnswEfConstruction() > 0 ? vectorIndexingOptions.getHnswEfConstruction() - : Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - return new Lucene95HnswVectorsFormat(m, efConstruction); + : Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + return new Lucene99HnswVectorsFormat(m, efConstruction); } /** @@ -134,10 +134,10 @@ protected void validateRequest(Field requestField) { } if (requestField.getSearch()) { - if (requestField.getVectorDimensions() > Lucene95HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS) { + if (requestField.getVectorDimensions() > Lucene99HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS) { throw new IllegalArgumentException( "Vector dimension must be <= " - + Lucene95HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS + + Lucene99HnswVectorsFormat.DEFAULT_MAX_DIMENSIONS + " for search"); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java index 44511b15f..9b2269270 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/BlendedTermQuery.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.TermQuery; @@ -78,21 +79,20 @@ public BlendedTermQuery(Term[] terms, float[] boosts) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } - IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; for (int i = 0; i < terms.length; i++) { - ctx[i] = TermStates.build(context, terms[i], true); + ctx[i] = TermStates.build(indexSearcher, terms[i], true); docFreqs[i] = ctx[i].docFreq(); } - final int maxDoc = reader.maxDoc(); - blend(ctx, maxDoc, reader); + final int maxDoc = indexSearcher.getIndexReader().maxDoc(); + blend(ctx, maxDoc, indexSearcher.getIndexReader()); return topLevelQuery(terms, ctx, docFreqs, maxDoc); } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java index f3a8cdde0..f574d91c7 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/MatchPhrasePrefixQuery.java @@ -42,6 +42,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; @@ -193,8 +194,8 @@ public MatchPhrasePrefixQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } @@ -211,7 +212,7 @@ public Query rewrite(IndexReader reader) throws IOException { int position = positions.get(sizeMinus1); Set terms = new HashSet<>(); for (Term term : suffixTerms) { - getPrefixTerms(terms, term, reader); + getPrefixTerms(terms, term, indexSearcher.getIndexReader()); if (terms.size() > maxExpansions) { break; } diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java index b73b9e9a4..5a7b6d974 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/FilterFunction.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; /** @@ -133,32 +134,30 @@ public float getWeight() { /** * Method to rewrite queries with the given {@link IndexReader}. Final to force use of {@link - * #doRewrite(IndexReader, boolean, Query)}. + * #doRewrite(boolean, Query)}. * - * @param reader index reader + * @param indexSearcher index searcher * @return function object with any query rewriting done * @throws IOException */ - public final FilterFunction rewrite(IndexReader reader) throws IOException { + public final FilterFunction rewrite(IndexSearcher indexSearcher) throws IOException { Query rewrittenFilterQuery = null; if (filterQuery != null) { - rewrittenFilterQuery = filterQuery.rewrite(reader); + rewrittenFilterQuery = filterQuery.rewrite(indexSearcher); } - return doRewrite(reader, rewrittenFilterQuery != filterQuery, rewrittenFilterQuery); + return doRewrite(rewrittenFilterQuery != filterQuery, rewrittenFilterQuery); } /** * Rewrite method for all child classes. * - * @param reader index reader * @param filterQueryRewritten if the filter query was modified by rewrite * @param rewrittenFilterQuery final value of rewritten query, may be null if no filter * @return fully rewritten filter function * @throws IOException */ protected abstract FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException; + boolean filterQueryRewritten, Query rewrittenFilterQuery) throws IOException; @Override public String toString() { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java index 934c32847..667472dff 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/MultiFunctionScoreQuery.java @@ -27,7 +27,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -124,16 +123,16 @@ public MultiFunctionScoreQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = super.rewrite(indexSearcher); if (rewritten != this) { return rewritten; } - Query rewrittenInner = innerQuery.rewrite(reader); + Query rewrittenInner = innerQuery.rewrite(indexSearcher); boolean needsRewrite = rewrittenInner != innerQuery; FilterFunction[] rewrittenFunctions = new FilterFunction[functions.length]; for (int i = 0; i < functions.length; ++i) { - rewrittenFunctions[i] = functions[i].rewrite(reader); + rewrittenFunctions[i] = functions[i].rewrite(indexSearcher); needsRewrite |= (rewrittenFunctions[i] != functions[i]); } if (needsRewrite) { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java index 809b58fc9..309ebb8a6 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/ScriptFilterFunction.java @@ -21,7 +21,6 @@ import com.yelp.nrtsearch.server.luceneserver.search.query.QueryUtils.SettableDoubleValues; import java.io.IOException; import java.util.Objects; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; @@ -54,9 +53,7 @@ public ScriptFilterFunction( } @Override - protected FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException { + protected FilterFunction doRewrite(boolean filterQueryRewritten, Query rewrittenFilterQuery) { if (filterQueryRewritten) { return new ScriptFilterFunction(rewrittenFilterQuery, getWeight(), script, scriptSource); } else { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java index 2c98768ae..dc9794860 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/search/query/multifunction/WeightFilterFunction.java @@ -16,7 +16,6 @@ package com.yelp.nrtsearch.server.luceneserver.search.query.multifunction; import java.io.IOException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Query; @@ -37,9 +36,7 @@ public WeightFilterFunction(Query filterQuery, float weight) { } @Override - protected FilterFunction doRewrite( - IndexReader reader, boolean filterQueryRewritten, Query rewrittenFilterQuery) - throws IOException { + protected FilterFunction doRewrite(boolean filterQueryRewritten, Query rewrittenFilterQuery) { if (filterQueryRewritten) { return new WeightFilterFunction(rewrittenFilterQuery, getWeight()); } else { diff --git a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java index cec15c1bb..67dce5742 100644 --- a/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java +++ b/src/main/java/com/yelp/nrtsearch/server/luceneserver/suggest/CompletionInfixSuggester.java @@ -24,7 +24,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; @@ -37,7 +37,7 @@ import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.suggest.InputIterator; import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.ContextQuery; import org.apache.lucene.search.suggest.document.ContextSuggestField; @@ -249,8 +249,8 @@ protected IndexWriterConfig getIndexWriterConfig( Analyzer indexAnalyzer, IndexWriterConfig.OpenMode mode) { IndexWriterConfig iwc = super.getIndexWriterConfig(indexAnalyzer, mode); Codec filterCodec = - new Lucene95Codec() { - final PostingsFormat fstPostingsFormat = new Completion90PostingsFormat(); + new Lucene99Codec() { + final PostingsFormat fstPostingsFormat = new Completion99PostingsFormat(); @Override public PostingsFormat getPostingsFormatForField(String field) { diff --git a/src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java similarity index 72% rename from src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java rename to src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java index 7da2411e9..485f18abf 100644 --- a/src/main/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java +++ b/src/main/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java @@ -20,32 +20,32 @@ import org.slf4j.LoggerFactory; /** - * Copy of the lucene Completion90PostingsFormat, but allows the FST load mode to be configured. + * Copy of the lucene Completion99PostingsFormat, but allows the FST load mode to be configured. * Since this codec is loaded by class name, it must have the same name as the original and be * present earlier in the class path. */ -public class Completion90PostingsFormat extends CompletionPostingsFormat { - private static final Logger logger = LoggerFactory.getLogger(Completion90PostingsFormat.class); +public class Completion99PostingsFormat extends CompletionPostingsFormat { + private static final Logger logger = LoggerFactory.getLogger(Completion99PostingsFormat.class); /** - * Creates a {@link Completion90PostingsFormat} that will load the completion FST based on the + * Creates a {@link Completion99PostingsFormat} that will load the completion FST based on the * value present in {@link CompletionPostingsFormatUtil}. */ - public Completion90PostingsFormat() { + public Completion99PostingsFormat() { this(CompletionPostingsFormatUtil.getCompletionCodecLoadMode()); } /** - * Creates a {@link Completion90PostingsFormat} that will use the provided fstLoadMode + * Creates a {@link Completion99PostingsFormat} that will use the provided fstLoadMode * to determine if the completion FST should be loaded on or off heap. */ - public Completion90PostingsFormat(FSTLoadMode fstLoadMode) { - super("Completion90", fstLoadMode); - logger.info("Created Completion90PostingsFormat with fstLoadMode: " + fstLoadMode); + public Completion99PostingsFormat(FSTLoadMode fstLoadMode) { + super("Completion99", fstLoadMode); + logger.info("Created Completion99PostingsFormat with fstLoadMode: " + fstLoadMode); } @Override protected PostingsFormat delegatePostingsFormat() { - return PostingsFormat.forName("Lucene90"); + return PostingsFormat.forName("Lucene99"); } } diff --git a/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java b/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java index f42da2a82..d5eea0063 100644 --- a/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java +++ b/src/main/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormatUtil.java @@ -24,7 +24,7 @@ public class CompletionPostingsFormatUtil { private CompletionPostingsFormatUtil() {} /** - * Set the FST load mode used by the modified {@link Completion90PostingsFormat}. Must be set + * Set the FST load mode used by the modified {@link Completion99PostingsFormat}. Must be set * before any index data is loaded. * * @param loadMode new FST load mode diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java index d1a80fa4e..540b09f60 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java @@ -214,7 +214,9 @@ public void testExplain() { var explain = hit.getExplain(); var expectedExplain = String.format( - "%d.0 = weight(FunctionScoreQuery(int_field:[0 TO 100], scored by expr(int_score))), result of:\n" + "%d.0 = weight(FunctionScoreQuery(IndexOrDocValuesQuery(" + + "indexQuery=int_field:[0 TO 100], dvQuery=int_field:[0 TO 100]), " + + "scored by expr(int_score))), result of:\n" + " % sizes, List deletions) { String.valueOf(id), sizes.get(i), false, + false, null, Collections.emptyMap(), new byte[StringHelper.ID_LENGTH],