Create runtime fields

clientlib/src/main/java/com/yelp/nrtsearch/server/utils/StructJsonUtils.java

@@ -71,6 +71,22 @@ public static Struct convertMapToStruct(Map<String, Object> map, boolean longAsS
     return builder.build();
   }
 
+  /**
+   * Convert an Iterable of native java types into a protobuf ListValue. This iterable may contain
+   * null, Boolean, String, Number, Iterable, or Map (String key).
+   *
+   * @param iterable iterable of java native types
+   * @param longAsString if Long values should be encoded as String
+   * @return protobuf list value for iterable
+   */
+  public static ListValue convertIterableToListValue(Iterable<?> iterable, boolean longAsString) {
+    ListValue.Builder listValueBuilder = ListValue.newBuilder();
+    for (Object e : iterable) {
+      listValueBuilder.addValues(convertObjectToValue(e, longAsString));
+    }
+    return listValueBuilder.build();
+  }
+
   /**
    * Convert an Iterable of native java types into a protobuf Value. This iterable may contain null,
    * Boolean, String, Number, Iterable, or Map (String key).

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -516,6 +516,8 @@ enum FieldType {
     // List of float values
     VECTOR = 15;
     CONTEXT_SUGGEST = 16; //Field used for contextual suggest fields
+    // Runtime fields
+    RUNTIME = 17;
 }
 
 //How the tokens should be indexed.
@@ -1132,4 +1134,4 @@ message CustomRequest {
 
 message CustomResponse {
     map<string, string> response = 1; // Custom response sent by the plugin
-}
+}

clientlib/src/main/proto/yelp/nrtsearch/search.proto

@@ -452,6 +452,8 @@ message SearchRequest {
     bool explain = 25;
     // Search nested object fields for each hit
     map<string, InnerHit> inner_hits = 26;
+    // Defines runtime fields for this query
+    repeated RuntimeField runtimeFields = 27;
 }
 
 /* Inner Hit search request */
@@ -478,6 +480,14 @@ message VirtualField {
     string name = 2; // Virtual field's name. Must be different from registered fields and any other virtual fields.
 }
 
+/* Runtime field used during search */
+message RuntimeField {
+    // Script defining this field's values.
+    Script script = 1;
+    // Runtime field's name. Must be different from registered fields and any other runtime fields.
+    string name = 2;
+}
+
 message Script {
     string lang = 1; // script language
     string source = 2; // script source
@@ -598,12 +608,12 @@ message SearchResponse {
                 google.protobuf.Struct structValue = 8; // Value for structured data
                 // Value for VECTOR FieldType
                 Vector vectorValue = 9;
+                google.protobuf.ListValue listValue = 10;
             }
 
             message Vector {
                 repeated float value = 1;
             }
-
         }
 
         message CompositeFieldValue {

src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java

@@ -37,15 +37,18 @@
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.ObjectFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.PolygonfieldDef;
+import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.innerhit.InnerHitFetchTask;
 import com.yelp.nrtsearch.server.luceneserver.rescore.RescoreTask;
+import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript;
 import com.yelp.nrtsearch.server.luceneserver.search.FieldFetchContext;
 import com.yelp.nrtsearch.server.luceneserver.search.SearchContext;
 import com.yelp.nrtsearch.server.luceneserver.search.SearchCutoffWrapper.CollectionTimeoutException;
 import com.yelp.nrtsearch.server.luceneserver.search.SearchRequestProcessor;
 import com.yelp.nrtsearch.server.luceneserver.search.SearcherResult;
 import com.yelp.nrtsearch.server.monitoring.SearchResponseCollector;
+import com.yelp.nrtsearch.server.utils.ObjectToCompositeFieldTransformer;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -763,7 +766,6 @@ private CompositeFieldValue getFieldForHit(
 
       // We detect invalid field above:
       assert fd != null;
-
       if (fd instanceof VirtualFieldDef) {
         VirtualFieldDef virtualFieldDef = (VirtualFieldDef) fd;
 
@@ -786,6 +788,18 @@ public boolean advanceExact(int doc) throws IOException {
         doubleValues.advanceExact(docID);
         compositeFieldValue.addFieldValue(
             FieldValue.newBuilder().setDoubleValue(doubleValues.doubleValue()));
+      } else if (fd instanceof RuntimeFieldDef) {
+        RuntimeFieldDef runtimeFieldDef = (RuntimeFieldDef) fd;
+        RuntimeScript.SegmentFactory segmentFactory = runtimeFieldDef.getSegmentFactory();
+        RuntimeScript values = segmentFactory.newInstance(leaf);
+        int docID = hit.getLuceneDocId() - leaf.docBase;
+        // Check if the value is available for the current document
+        if (values != null) {
+          values.setDocId(docID);
+          Object obj = values.execute();
+          ObjectToCompositeFieldTransformer.enrichCompositeField(obj, compositeFieldValue);
+        }
+
       } else if (fd instanceof IndexableFieldDef && ((IndexableFieldDef) fd).hasDocValues()) {
         int docID = hit.getLuceneDocId() - leaf.docBase;
         // it may be possible to cache this if there are multiple hits in the same segment
@@ -794,6 +808,7 @@ public boolean advanceExact(int doc) throws IOException {
         for (int i = 0; i < docValues.size(); ++i) {
           compositeFieldValue.addFieldValue(docValues.toFieldValue(i));
         }
+
       }
       // retrieve stored fields
       else if (fd instanceof IndexableFieldDef && ((IndexableFieldDef) fd).isStored()) {
@@ -896,6 +911,12 @@ private static void fetchSlice(
               sliceSegment,
               fieldDefEntry.getKey(),
               (VirtualFieldDef) fieldDefEntry.getValue());
+        } else if (fieldDefEntry.getValue() instanceof RuntimeFieldDef) {
+          fetchRuntimeFromSegmentFactory(
+              sliceHits,
+              sliceSegment,
+              fieldDefEntry.getKey(),
+              (RuntimeFieldDef) fieldDefEntry.getValue());
         } else if (fieldDefEntry.getValue() instanceof IndexableFieldDef) {
           IndexableFieldDef indexableFieldDef = (IndexableFieldDef) fieldDefEntry.getValue();
           if (indexableFieldDef.hasDocValues()) {
@@ -951,6 +972,30 @@ private static void fetchFromValueSource(
       }
     }
 
+    /** Fetch field value from runtime field's Object. */
+    private static void fetchRuntimeFromSegmentFactory(
+        List<SearchResponse.Hit.Builder> sliceHits,
+        LeafReaderContext sliceSegment,
+        String name,
+        RuntimeFieldDef runtimeFieldDef)
+        throws IOException {
+      RuntimeScript.SegmentFactory segmentFactory = runtimeFieldDef.getSegmentFactory();
+      RuntimeScript values = segmentFactory.newInstance(sliceSegment);
+
+      for (SearchResponse.Hit.Builder hit : sliceHits) {
+        int docID = hit.getLuceneDocId() - sliceSegment.docBase;
+        // Check if the value is available for the current document
+        if (values != null) {
+          values.setDocId(docID);
+          Object obj = values.execute();
+          SearchResponse.Hit.CompositeFieldValue.Builder compositeFieldValue =
+              SearchResponse.Hit.CompositeFieldValue.newBuilder();
+          ObjectToCompositeFieldTransformer.enrichCompositeField(obj, compositeFieldValue);
+          hit.putFields(name, compositeFieldValue.build());
+        }
+      }
+    }
+
     /** Fetch field value from its doc value */
     private static void fetchFromDocVales(
         List<SearchResponse.Hit.Builder> sliceHits,

src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java

@@ -28,6 +28,7 @@
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.IntFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.LongFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.script.FacetScript;
 import com.yelp.nrtsearch.server.luceneserver.script.ScriptService;
@@ -324,10 +325,11 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult(
     }
 
     FacetResult facetResult;
-    if (!(fieldDef instanceof IndexableFieldDef) && !(fieldDef instanceof VirtualFieldDef)) {
+    if (!(fieldDef instanceof IndexableFieldDef)
+        && !(fieldDef instanceof VirtualFieldDef || fieldDef instanceof RuntimeFieldDef)) {
       throw new IllegalArgumentException(
           String.format(
-              "field %s is neither a virtual field nor registered as an indexable field. Facets are supported only for these types",
+              "field %s is neither a virtual/runtime field nor registered as an indexable field. Facets are supported only for these types",
               fieldName));
     }
     if (!facet.getNumericRangeList().isEmpty()) {

src/main/java/com/yelp/nrtsearch/server/luceneserver/field/FieldDefCreator.java

@@ -55,6 +55,11 @@ public FieldDefCreator(LuceneServerConfiguration configuration) {
         (name, field) -> {
           throw new UnsupportedOperationException("Virtual fields should be created directly");
         });
+    register(
+        "RUNTIME",
+        (name, field) -> {
+          throw new UnsupportedOperationException("Runtime fields should be created directly");
+        });
     register("VECTOR", VectorFieldDef::new);
     register("CONTEXT_SUGGEST", ContextSuggestFieldDef::new);
   }

src/main/java/com/yelp/nrtsearch/server/luceneserver/field/RuntimeFieldDef.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Yelp Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yelp.nrtsearch.server.luceneserver.field;
+
+import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef.FacetValueType;
+import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript;
+
+/**
+ * Field definition for a runtime field. Runtime fields are able to produce a value for each given
+ * index document.
+ */
+public class RuntimeFieldDef extends FieldDef {
+  private final RuntimeScript.SegmentFactory segmentFactory;
+  private final IndexableFieldDef.FacetValueType facetValueType;
+
+  /**
+   * Field constructor.
+   *
+   * @param name name of field
+   * @param segmentFactory lucene value source used to produce field value from documents
+   */
+  public RuntimeFieldDef(String name, RuntimeScript.SegmentFactory segmentFactory) {
+    super(name);
+    this.segmentFactory = segmentFactory;
+    this.facetValueType = FacetValueType.NO_FACETS;
+  }
+
+  /**
+   * Get value source for this field.
+   *
+   * @return Segment factory to create the expression.
+   */
+  public RuntimeScript.SegmentFactory getSegmentFactory() {
+    return segmentFactory;
+  }
+
+  @Override
+  public String getType() {
+    return "RUNTIME";
+  }
+
+  /**
+   * Get the facet value type for this field.
+   *
+   * @return field facet value type
+   */
+  @Override
+  public IndexableFieldDef.FacetValueType getFacetValueType() {
+    return facetValueType;
+  }
+}

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/FieldUpdateHandler.java

@@ -119,6 +119,7 @@ public static UpdatedFieldInfo updateFields(
       parseVirtualField(field, fieldStateBuilder);
       newFields.put(field.getName(), field);
     }
+
     return new UpdatedFieldInfo(newFields, fieldStateBuilder.build());
   }
 

src/main/java/com/yelp/nrtsearch/server/luceneserver/script/RuntimeScript.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2020 Yelp Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yelp.nrtsearch.server.luceneserver.script;
+
+import com.yelp.nrtsearch.server.luceneserver.doc.DocLookup;
+import com.yelp.nrtsearch.server.luceneserver.doc.LoadedDocValues;
+import com.yelp.nrtsearch.server.luceneserver.doc.SegmentDocLookup;
+import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript.SegmentFactory;
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.index.LeafReaderContext;
+
+/**
+ * Script to produce a value for a given document. Implementations must have an execute function.
+ * This class conforms with the script compile contract, see {@link ScriptContext}. The script has
+ * access to the query parameters, the document doc values through {@link SegmentDocLookup}.
+ */
+public abstract class RuntimeScript {
+  private final Map<String, Object> params;
+  private final SegmentDocLookup segmentDocLookup;
+
+  // names for parameters to execute
+  public static final String[] PARAMETERS = new String[] {};
+
+  /**
+   * ObjectScript constructor.
+   *
+   * @param params script parameters from {@link com.yelp.nrtsearch.server.grpc.Script}
+   * @param docLookup index level doc values lookup
+   * @param leafContext lucene segment context
+   */
+  public RuntimeScript(
+      Map<String, Object> params, DocLookup docLookup, LeafReaderContext leafContext) {
+    this.params = params;
+    this.segmentDocLookup = docLookup.getSegmentLookup(leafContext);
+  }
+
+  /**
+   * Main script function.
+   *
+   * @return object value computed for document
+   */
+  public abstract Object execute();
+
+  /** Set segment level id for the next document to score. * */
+  public void setDocId(int docId) {
+    segmentDocLookup.setDocId(docId);
+  }
+
+  /** Get the script parameters provided in the request. */
+  public Map<String, Object> getParams() {
+    return params;
+  }
+
+  /** Get doc values for the current document. */
+  public Map<String, LoadedDocValues<?>> getDoc() {
+    return segmentDocLookup;
+  }
+
+  /** Factory interface for creating a RuntimeScript bound to a lucene segment. */
+  public interface SegmentFactory {
+
+    /**
+     * Create a RuntimeScript instance for a lucene segment.
+     *
+     * @param context lucene segment context
+     * @return segment level RuntimeScript
+     * @throws IOException
+     */
+    RuntimeScript newInstance(LeafReaderContext context) throws IOException;
+  }
+
+  /**
+   * Factory required for the compilation of a RuntimeScript. Used to produce request level {@link
+   * SegmentFactory}. See script compile contract {@link ScriptContext}.
+   */
+  public interface Factory {
+    /**
+     * Create request level {@link RuntimeScript.SegmentFactory}.
+     *
+     * @param params parameters from script request
+     * @param docLookup index level doc value lookup provider
+     * @return {@link RuntimeScript.SegmentFactory} to evaluate script
+     */
+    SegmentFactory newFactory(Map<String, Object> params, DocLookup docLookup);
+  }
+
+  // compile context for the RuntimeScript, contains script type info
+  public static final ScriptContext<Factory> CONTEXT =
+      new ScriptContext<>("runtime", Factory.class, SegmentFactory.class, RuntimeScript.class);
+}

src/main/java/com/yelp/nrtsearch/server/luceneserver/script/ScriptService.java

@@ -47,6 +47,7 @@ public class ScriptService {
       ImmutableList.<ScriptContext<?>>builder()
           .add(FacetScript.CONTEXT)
           .add(ScoreScript.CONTEXT)
+          .add(RuntimeScript.CONTEXT)
           .build();
 
   private final Map<String, ScriptEngine> scriptEngineMap = new HashMap<>();