Skip to content

Commit

Permalink
Refactor verbose index metrics to search response metrics (#631)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aprudhomme
aprudhomme authored Mar 12, 2024
1 parent 4b7eccc commit 978b7f6
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 54 deletions.
2 changes: 1 addition & 1 deletion src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ private void registerMetrics(GlobalState globalState) {
new DirSizeCollector(globalState).register(collectorRegistry);
new ProcStatCollector().register(collectorRegistry);
new MergeSchedulerCollector(globalState).register(collectorRegistry);
new VerboseIndexCollector(globalState).register(collectorRegistry);
new SearchResponseCollector(globalState).register(collectorRegistry);
}

/** Main launches the server from the command line. */
Expand Down
10 changes: 6 additions & 4 deletions src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import com.yelp.nrtsearch.server.luceneserver.search.SearchCutoffWrapper.CollectionTimeoutException;
import com.yelp.nrtsearch.server.luceneserver.search.SearchRequestProcessor;
import com.yelp.nrtsearch.server.luceneserver.search.SearcherResult;
import com.yelp.nrtsearch.server.monitoring.VerboseIndexCollector;
import com.yelp.nrtsearch.server.monitoring.SearchResponseCollector;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
Expand Down Expand Up @@ -290,9 +290,11 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest)
// if we are out of time, don't bother with serialization
DeadlineUtils.checkDeadline("SearchHandler: end", diagnostics, "SEARCH");
SearchResponse searchResponse = searchContext.getResponseBuilder().build();
if (!warming && searchContext.getIndexState().getVerboseMetrics()) {
VerboseIndexCollector.updateSearchResponseMetrics(
searchResponse, searchContext.getIndexState().getName());
if (!warming) {
SearchResponseCollector.updateSearchResponseMetrics(
searchResponse,
searchContext.getIndexState().getName(),
searchContext.getIndexState().getVerboseMetrics());
}
return searchResponse;
}
Expand Down
51 changes: 28 additions & 23 deletions ...ver/monitoring/VerboseIndexCollector.java → ...r/monitoring/SearchResponseCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.slf4j.LoggerFactory;

/**
* Collector for detailed index metrics that may be expensive to produce or publish. Collection can
* be toggled with the verbose metrics index live setting.
* Collector for metrics related to the {@link SearchResponse}. Has the option to collect verbose
* metrics, which may be expensive to produce or publish
*/
public class VerboseIndexCollector extends Collector {
private static final Logger logger = LoggerFactory.getLogger(VerboseIndexCollector.class);
public class SearchResponseCollector extends Collector {
private static final Logger logger = LoggerFactory.getLogger(SearchResponseCollector.class);
private final GlobalState globalState;

private static final Summary searchResponseSizeBytes =
Expand Down Expand Up @@ -86,29 +86,33 @@ public class VerboseIndexCollector extends Collector {
.labelNames("index")
.create();

public static void updateSearchResponseMetrics(SearchResponse searchResponse, String index) {
searchResponseSizeBytes.labels(index).observe(searchResponse.getSerializedSize());
searchResponseTotalHits.labels(index).observe(searchResponse.getTotalHits().getValue());
public static void updateSearchResponseMetrics(
SearchResponse searchResponse, String index, boolean verbose) {
if (searchResponse.getHitTimeout()) {
searchTimeoutCount.labels(index).inc();
}
if (searchResponse.getTerminatedEarly()) {
searchTerminatedEarlyCount.labels(index).inc();
}

Diagnostics diagnostics = searchResponse.getDiagnostics();
searchStageLatencyMs.labels(index, "recall").observe(diagnostics.getFirstPassSearchTimeMs());
searchStageLatencyMs.labels(index, "highlight").observe(diagnostics.getHighlightTimeMs());
searchStageLatencyMs.labels(index, "fetch").observe(diagnostics.getGetFieldsTimeMs());
for (Map.Entry<String, Double> entry : diagnostics.getFacetTimeMsMap().entrySet()) {
searchStageLatencyMs.labels(index, "facet:" + entry.getKey()).observe(entry.getValue());
}
for (Map.Entry<String, Double> entry : diagnostics.getRescorersTimeMsMap().entrySet()) {
searchStageLatencyMs.labels(index, "rescorer:" + entry.getKey()).observe(entry.getValue());
if (verbose) {
searchResponseSizeBytes.labels(index).observe(searchResponse.getSerializedSize());
searchResponseTotalHits.labels(index).observe(searchResponse.getTotalHits().getValue());

Diagnostics diagnostics = searchResponse.getDiagnostics();
searchStageLatencyMs.labels(index, "recall").observe(diagnostics.getFirstPassSearchTimeMs());
searchStageLatencyMs.labels(index, "highlight").observe(diagnostics.getHighlightTimeMs());
searchStageLatencyMs.labels(index, "fetch").observe(diagnostics.getGetFieldsTimeMs());
for (Map.Entry<String, Double> entry : diagnostics.getFacetTimeMsMap().entrySet()) {
searchStageLatencyMs.labels(index, "facet:" + entry.getKey()).observe(entry.getValue());
}
for (Map.Entry<String, Double> entry : diagnostics.getRescorersTimeMsMap().entrySet()) {
searchStageLatencyMs.labels(index, "rescorer:" + entry.getKey()).observe(entry.getValue());
}
}
}

public VerboseIndexCollector(GlobalState globalState) {
public SearchResponseCollector(GlobalState globalState) {
this.globalState = globalState;
}

Expand All @@ -117,23 +121,24 @@ public List<MetricFamilySamples> collect() {
List<MetricFamilySamples> mfs = new ArrayList<>();

try {
boolean publishMetrics = false;
mfs.addAll(searchTimeoutCount.collect());
mfs.addAll(searchTerminatedEarlyCount.collect());

boolean publishVerboseMetrics = false;
Set<String> indexNames = globalState.getIndexNames();
for (String indexName : indexNames) {
if (globalState.getIndex(indexName).getVerboseMetrics()) {
publishMetrics = true;
publishVerboseMetrics = true;
break;
}
}
if (publishMetrics) {
if (publishVerboseMetrics) {
mfs.addAll(searchResponseSizeBytes.collect());
mfs.addAll(searchResponseTotalHits.collect());
mfs.addAll(searchStageLatencyMs.collect());
mfs.addAll(searchTimeoutCount.collect());
mfs.addAll(searchTerminatedEarlyCount.collect());
}
} catch (Exception e) {
logger.warn("Error getting verbose index metrics: ", e);
logger.warn("Error getting search response metrics: ", e);
}

return mfs;
Expand Down
55 changes: 29 additions & 26 deletions ...monitoring/VerboseIndexCollectorTest.java → ...nitoring/SearchResponseCollectorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package com.yelp.nrtsearch.server.monitoring;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand All @@ -32,17 +31,26 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.BeforeClass;
import org.junit.Test;

public class VerboseIndexCollectorTest {
public class SearchResponseCollectorTest {

@Test
public void testNoIndex() {
GlobalState mockGlobalState = mock(GlobalState.class);
when(mockGlobalState.getIndexNames()).thenReturn(Collections.emptySet());

VerboseIndexCollector collector = new VerboseIndexCollector(mockGlobalState);
assertTrue(collector.collect().isEmpty());
@BeforeClass
public static void init() {
SearchResponse testResponse =
SearchResponse.newBuilder()
.setTotalHits(TotalHits.newBuilder().setValue(10).build())
.setDiagnostics(
Diagnostics.newBuilder()
.setFirstPassSearchTimeMs(1.0)
.setHighlightTimeMs(2.0)
.setGetFieldsTimeMs(3.0)
.putFacetTimeMs("facet1", 4.0)
.putRescorersTimeMs("rescorer", 5.0)
.build())
.build();
SearchResponseCollector.updateSearchResponseMetrics(testResponse, "test_index", true);
}

@Test
Expand All @@ -54,34 +62,29 @@ public void testVerboseMetricsDisabled() throws IOException {
when(mockGlobalState.getIndexNames()).thenReturn(Collections.singleton("test_index"));
when(mockGlobalState.getIndex("test_index")).thenReturn(mockIndexState);

VerboseIndexCollector collector = new VerboseIndexCollector(mockGlobalState);
assertTrue(collector.collect().isEmpty());
SearchResponseCollector collector = new SearchResponseCollector(mockGlobalState);
List<MetricFamilySamples> metrics = collector.collect();
assertEquals(2, metrics.size());

Map<String, MetricFamilySamples> metricsMap = new HashMap<>();
for (MetricFamilySamples samples : metrics) {
metricsMap.put(samples.name, samples);
}
assertEquals(
Set.of("nrt_search_timeout_count", "nrt_search_terminated_early_count"),
metricsMap.keySet());
}

@Test
public void testVerboseMetricsEnabled() throws IOException {
SearchResponse testResponse =
SearchResponse.newBuilder()
.setTotalHits(TotalHits.newBuilder().setValue(10).build())
.setDiagnostics(
Diagnostics.newBuilder()
.setFirstPassSearchTimeMs(1.0)
.setHighlightTimeMs(2.0)
.setGetFieldsTimeMs(3.0)
.putFacetTimeMs("facet1", 4.0)
.putRescorersTimeMs("rescorer", 5.0)
.build())
.build();
VerboseIndexCollector.updateSearchResponseMetrics(testResponse, "test_index");

GlobalState mockGlobalState = mock(GlobalState.class);
IndexState mockIndexState = mock(IndexState.class);

when(mockIndexState.getVerboseMetrics()).thenReturn(true);
when(mockGlobalState.getIndexNames()).thenReturn(Collections.singleton("test_index"));
when(mockGlobalState.getIndex("test_index")).thenReturn(mockIndexState);

VerboseIndexCollector collector = new VerboseIndexCollector(mockGlobalState);
SearchResponseCollector collector = new SearchResponseCollector(mockGlobalState);
List<MetricFamilySamples> metrics = collector.collect();
assertEquals(5, metrics.size());

Expand Down

0 comments on commit 978b7f6

Please sign in to comment.