Use direct executor for DrillSideways tasks (#651)

* Use direct executor for DrillSideways tasks

* Add comment

src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java

@@ -16,6 +16,7 @@
 package com.yelp.nrtsearch.server.luceneserver;
 
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.MoreExecutors;
 import com.google.protobuf.Struct;
 import com.google.protobuf.util.JsonFormat;
 import com.yelp.nrtsearch.server.grpc.DeadlineUtils;
@@ -53,10 +54,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.*;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -77,6 +75,7 @@
 import org.slf4j.LoggerFactory;
 
 public class SearchHandler implements Handler<SearchRequest, SearchResponse> {
+  private static final ExecutorService DIRECT_EXECUTOR = MoreExecutors.newDirectExecutorService();
 
   private static final Logger logger = LoggerFactory.getLogger(SearchHandler.class);
   private final ThreadPoolExecutor threadPoolExecutor;
@@ -137,6 +136,11 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest)
         DrillDownQuery ddq = (DrillDownQuery) searchContext.getQuery();
 
         List<FacetResult> grpcFacetResults = new ArrayList<>();
+        // Run the drill sideways search on the direct executor to run subtasks in the
+        // current (grpc) thread. If we use the search thread pool for this, it can cause a
+        // deadlock trying to execute the dependent parallel search tasks. Since we do not
+        // currently add additional drill down definitions, there will only be one drill
+        // sideways task per query.
         DrillSideways drillS =
             new DrillSidewaysImpl(
                 s.searcher,
@@ -148,7 +152,7 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest)
                 shardState,
                 searchContext.getQueryFields(),
                 grpcFacetResults,
-                threadPoolExecutor,
+                DIRECT_EXECUTOR,
                 diagnostics);
         DrillSideways.ConcurrentDrillSidewaysResult<SearcherResult> concurrentDrillSidewaysResult;
         try {