diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index bd04156ba16..0d3b41e955a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -87,6 +87,9 @@ Bug Fixes
   all the cores.
   (Danyal Prout via shalin)
 
+* SOLR-9882: 500 error code on breaching timeAllowed by core and distributed (fsv) search, 
+  old and json facets (Mikhail Khludnev)
+
 Improvements
 ----------------------
 * SOLR-12999: Index replication could delete segments before downloading segments from master if there is not enough
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index 986c286aa37..ba94104bba1 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -54,11 +54,11 @@ import org.apache.solr.common.util.SolrjNamedThreadFactory;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.TimeOut;
 import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory;
 import org.eclipse.jetty.http2.HTTP2Cipher;
 import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory;
 import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
-import org.apache.solr.util.TimeOut;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -66,6 +66,7 @@ import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.server.session.DefaultSessionIdManager;
 import org.eclipse.jetty.servlet.FilterHolder;
@@ -333,6 +334,8 @@ public class JettySolrRunner {
       server.setConnectors(new Connector[] {connector});
     }
 
+    HandlerWrapper chain;
+    {
     // Initialize the servlets
     final ServletContextHandler root = new ServletContextHandler(server, config.context, ServletContextHandler.SESSIONS);
 
@@ -391,11 +394,15 @@ public class JettySolrRunner {
         System.clearProperty("hostPort");
       }
     });
-
     // for some reason, there must be a servlet for this to get applied
     root.addServlet(Servlet404.class, "/*");
+    chain = root;
+    }
+
+    chain = injectJettyHandlers(chain);
+    
     GzipHandler gzipHandler = new GzipHandler();
-    gzipHandler.setHandler(root);
+    gzipHandler.setHandler(chain);
 
     gzipHandler.setMinGzipSize(0);
     gzipHandler.setCheckGzExists(false);
@@ -406,6 +413,13 @@ public class JettySolrRunner {
     server.setHandler(gzipHandler);
   }
 
+  /** descendants may inject own handler chaining it to the given root 
+   * and then returning that own one*/
+  protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
+    return chain;
+  }
+
+
   /**
    * @return the {@link SolrDispatchFilter} for this node
    */
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index a398eb73d3e..ee7923f4b00 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -200,9 +200,8 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
       // count timeouts
       NamedList header = rsp.getResponseHeader();
       if(header != null) {
-        Object partialResults = header.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY);
-        boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
-        if( timedOut ) {
+        if( Boolean.TRUE.equals(header.getBooleanArg(
+                     SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY)) ) {
           numTimeouts.mark();
           rsp.setHttpCaching(false);
         }
diff --git a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
index e01c9582f19..e339a756a36 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
@@ -47,6 +47,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.request.SimpleFacets;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.schema.PointField;
 import org.apache.solr.search.QueryParsing;
@@ -711,6 +712,17 @@ public class FacetComponent extends SearchComponent {
       NamedList facet_counts = null;
       try {
         facet_counts = (NamedList) srsp.getSolrResponse().getResponse().get("facet_counts");
+        if (facet_counts==null) {
+          NamedList<?> responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+          if (Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+            continue;
+          } else {
+            log.warn("corrupted response on "+srsp.getShardRequest()+": "+srsp.getSolrResponse());
+            throw new SolrException(ErrorCode.SERVER_ERROR,
+                "facet_counts is absent in response from " + srsp.getNodeName() +
+                ", but "+SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" hasn't been responded");
+          }
+        }
       } catch (Exception ex) {
         if (ShardParams.getShardsTolerantAsBool(rb.req.getParams())) {
           continue; // looks like a shard did not return anything
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index e9373702897..4dab304b5f3 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import org.apache.lucene.index.ExitableDirectoryReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
@@ -384,6 +385,7 @@ public class QueryComponent extends SearchComponent
     // TODO: See SOLR-5595
     boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
     if(fsv){
+      try {
       NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields
       IndexReaderContext topReaderContext = searcher.getTopReaderContext();
       List<LeafReaderContext> leaves = topReaderContext.leaves();
@@ -394,13 +396,12 @@ public class QueryComponent extends SearchComponent
         leaves=null;
       }
 
-      DocList docList = rb.getResults().docList;
+      final DocList docs = rb.getResults().docList;
 
       // sort ids from lowest to highest so we can access them in order
-      int nDocs = docList.size();
+      int nDocs = docs.size();
       final long[] sortedIds = new long[nDocs];
       final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds
-      DocList docs = rb.getResults().docList;
       DocIterator it = docs.iterator();
       for (int i=0; i<nDocs; i++) {
         sortedIds[i] = (((long)it.nextDoc()) << 32) | i;
@@ -476,8 +477,13 @@ public class QueryComponent extends SearchComponent
 
         sortVals.add(sortField.getField(), vals);
       }
-
       rsp.add("sort_values", sortVals);
+    }catch(ExitableDirectoryReader.ExitingReaderException x) {
+      // it's hard to understand where we stopped, so yield nothing
+      // search handler will flag partial results
+      rsp.add("sort_values",new NamedList<>() );
+      throw x;
+    }
     }
   }
 
@@ -860,7 +866,7 @@ public class QueryComponent extends SearchComponent
         }
 
         if (responseHeader != null) {
-          if (Boolean.TRUE.equals(responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+          if (Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
             partialResults = true;
           }
           if (!Boolean.TRUE.equals(segmentTerminatedEarly)) {
@@ -880,6 +886,9 @@ public class QueryComponent extends SearchComponent
         numFound += docs.getNumFound();
 
         NamedList sortFieldValues = (NamedList)(srsp.getSolrResponse().getResponse().get("sort_values"));
+        if (sortFieldValues.size()==0 && partialResults) {
+          continue; //fsv timeout yields empty sort_vlaues
+        }
         NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
 
         // go through every doc in this response, construct a ShardDoc, and
@@ -958,9 +967,8 @@ public class QueryComponent extends SearchComponent
       populateNextCursorMarkFromMergedShards(rb);
 
       if (partialResults) {
-        if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
-          rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
-        }
+         rb.rsp.getResponseHeader().asShallowMap()
+                   .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
       }
       if (segmentTerminatedEarly != null) {
         final Object existingSegmentTerminatedEarly = rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
@@ -1158,6 +1166,13 @@ public class QueryComponent extends SearchComponent
           
           continue;
         }
+        {
+          NamedList<?> responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+          if (responseHeader!=null && Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+            rb.rsp.getResponseHeader().asShallowMap()
+               .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+          }
+        }
         SolrDocumentList docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
         for (SolrDocument doc : docs) {
           Object id = doc.getFieldValue(keyFieldName);
@@ -1436,7 +1451,7 @@ public class QueryComponent extends SearchComponent
 
     ResultContext ctx = new BasicResultContext(rb);
     rsp.addResponse(ctx);
-    rsp.getToLog().add("hits", rb.getResults().docList.matches());
+    rsp.getToLog().add("hits", rb.getResults()==null || rb.getResults().docList==null ? 0 : rb.getResults().docList.matches());
 
     if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
       if (null != rb.getNextCursorMark()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
index 1f9e2d51183..f757ad73bf1 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
@@ -30,6 +30,7 @@ import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.CursorMark;
 import org.apache.solr.search.DocListAndSet;
+import org.apache.solr.search.DocSlice;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.QueryCommand;
 import org.apache.solr.search.QueryResult;
@@ -460,7 +461,11 @@ public class ResponseBuilder
   public void setResult(QueryResult result) {
     setResults(result.getDocListAndSet());
     if (result.isPartialResults()) {
-      rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+      rsp.getResponseHeader().asShallowMap()
+          .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+      if(getResults().docList==null) {
+        getResults().docList = new DocSlice(0, 0, new int[] {}, new float[] {}, 0, 0);
+      }
     }
     final Boolean segmentTerminatedEarly = result.getSegmentTerminatedEarly();
     if (segmentTerminatedEarly != null) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index d4c680cffca..89cfa2ee53b 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -16,6 +16,9 @@
  */
 package org.apache.solr.handler.component;
 
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+import static org.apache.solr.common.params.CommonParams.PATH;
+
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.lang.invoke.MethodHandles;
@@ -53,9 +56,6 @@ import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.params.CommonParams.DISTRIB;
-import static org.apache.solr.common.params.CommonParams.PATH;
-
 
 /**
  *
@@ -315,17 +315,16 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
         }
       } catch (ExitableDirectoryReader.ExitingReaderException ex) {
         log.warn( "Query: " + req.getParamString() + "; " + ex.getMessage());
-        SolrDocumentList r = (SolrDocumentList) rb.rsp.getResponse();
-        if(r == null)
-          r = new SolrDocumentList();
-        r.setNumFound(0);
-        rb.rsp.addResponse(r);
+        if( rb.rsp.getResponse() == null) {
+          rb.rsp.addResponse(new SolrDocumentList());
+        }
         if(rb.isDebug()) {
           NamedList debug = new NamedList();
           debug.add("explain", new NamedList());
           rb.rsp.add("debug", debug);
         }
-        rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+        rb.rsp.getResponseHeader().asShallowMap()
+              .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
       } finally {
         SolrQueryTimeoutImpl.reset();
       }
@@ -413,9 +412,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
                   throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException());
                 }
               } else {
-                if(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
-                  rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
-                }
+                rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
               }
             }
 
diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
index 1bea80af07d..35ef0fe59f1 100644
--- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
@@ -38,6 +38,7 @@ import java.util.concurrent.Semaphore;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
+import org.apache.lucene.index.ExitableDirectoryReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.MultiPostingsEnum;
@@ -828,7 +829,11 @@ public class SimpleFacets {
             return result;
           } catch (SolrException se) {
             throw se;
-          } catch (Exception e) {
+          } 
+          catch(ExitableDirectoryReader.ExitingReaderException timeout) {
+            throw timeout;
+          }
+          catch (Exception e) {
             throw new SolrException(ErrorCode.SERVER_ERROR,
                                     "Exception during facet.field: " + facetValue, e);
           } finally {
diff --git a/solr/core/src/java/org/apache/solr/response/BasicResultContext.java b/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
index 792e483c548..579c0ad7b81 100644
--- a/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
+++ b/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
@@ -31,6 +31,7 @@ public class BasicResultContext extends ResultContext {
   private SolrQueryRequest req;
 
   public BasicResultContext(DocList docList, ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) {
+    assert docList!=null;
     this.docList = docList;
     this.returnFields = returnFields;
     this.searcher = searcher;
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index a659d6631bd..ab0996d31df 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -1713,7 +1713,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
       set = DocSetUtil.getDocSet(setCollector, this);
 
       totalHits = topCollector.getTotalHits();
-      assert (totalHits == set.size());
+      assert (totalHits == set.size()) || qr.isPartialResults();
 
       TopDocs topDocs = topCollector.topDocs(0, len);
       if (cmd.getSort() != null && query instanceof RankQuery == false && (cmd.getFlags() & GET_SCORES) != 0) {
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
index a5d55fbe604..5da2cefab2c 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
@@ -36,6 +36,7 @@ import org.apache.solr.handler.component.ResponseBuilder;
 import org.apache.solr.handler.component.SearchComponent;
 import org.apache.solr.handler.component.ShardRequest;
 import org.apache.solr.handler.component.ShardResponse;
+import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.QueryContext;
 import org.noggit.CharArr;
 import org.noggit.JSONWriter;
@@ -142,7 +143,8 @@ public class FacetModule extends SearchComponent {
       rb.req.getContext().put("FacetDebugInfo", fdebug);
     }
 
-    final Object results = facetState.facetRequest.process(fcontext);
+    Object results = facetState.facetRequest.process(fcontext);
+    // ExitableDirectory timeout causes absent "facets"
     rb.rsp.add("facets", results);
   }
 
@@ -167,7 +169,7 @@ public class FacetModule extends SearchComponent {
     }
 
     // Check if there are any refinements possible
-    if (facetState.mcontext.getSubsWithRefinement(facetState.facetRequest).isEmpty()) {
+    if ((facetState.mcontext==null) ||facetState.mcontext.getSubsWithRefinement(facetState.facetRequest).isEmpty()) {
       clearFaceting(rb.outgoing);
       return ResponseBuilder.STAGE_DONE;
     }
@@ -277,7 +279,13 @@ public class FacetModule extends SearchComponent {
       NamedList<Object> top = rsp.getResponse();
       if (top == null) continue; // shards.tolerant=true will cause this to happen on exceptions/errors
       Object facet = top.get("facets");
-      if (facet == null) continue;
+      if (facet == null) {
+        SimpleOrderedMap shardResponseHeader = (SimpleOrderedMap)rsp.getResponse().get("responseHeader");
+        if(Boolean.TRUE.equals(shardResponseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+          rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+        }
+        continue;
+      }
       if (facetState.merger == null) {
         facetState.merger = facetState.facetRequest.createFacetMerger(facet);
         facetState.mcontext = new FacetMerger.Context( sreq.responses.size() );
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
index 752a71e8118..fd8ce79273e 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
@@ -407,11 +407,14 @@ public abstract class FacetRequest {
       debugInfo.setProcessor(facetProcessor.getClass().getSimpleName());
       debugInfo.putInfoItem("domainSize", (long) fcontext.base.size());
       RTimer timer = new RTimer();
-      facetProcessor.process();
-      debugInfo.setElapse((long) timer.getTime());
+      try {
+        facetProcessor.process();
+      }finally {
+        debugInfo.setElapse((long) timer.getTime());
+      }
     }
 
-    return facetProcessor.getResponse(); // note: not captured in elapsed time above; good/bad?
+    return facetProcessor.getResponse(); 
   }
 
   public abstract FacetProcessor createFacetProcessor(FacetContext fcontext);
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
index edad958b399..163c38d6da8 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
@@ -100,9 +100,7 @@ public class SearchGroupShardResponseProcessor implements ShardResponseProcessor
         shardInfo.add(srsp.getShard(), nl);
       }
       if (ShardParams.getShardsTolerantAsBool(rb.req.getParams()) && srsp.getException() != null) {
-        if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
-          rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
-        }
+        rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
         continue; // continue if there was an error and we're tolerant.
       }
       maxElapsedTime = (int) Math.max(maxElapsedTime, srsp.getSolrResponse().getElapsedTime());
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
index e2dd2990b70..2db6b22f9ee 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
@@ -111,9 +111,7 @@ public class TopGroupsShardResponseProcessor implements ShardResponseProcessor {
         shardInfo.add(srsp.getShard(), individualShardInfo);
       }
       if (ShardParams.getShardsTolerantAsBool(rb.req.getParams()) && srsp.getException() != null) {
-        if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
-          rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
-        }
+        rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
         continue; // continue if there was an error and we're tolerant.  
       }
       NamedList<NamedList> secondPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("secondPhase");
diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
index 52a63f41944..72f148c8f44 100644
--- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
@@ -24,5 +24,6 @@
   <field name="_version_" type="long" indexed="true" stored="true"/>
   <field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
   <field name="id" type="string" indexed="true" stored="true"/>
+  <field name="num"  type="int"  indexed="true"  stored="true"/>
   <uniqueKey>id</uniqueKey>
 </schema>
diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
index af1036f999f..5d466e82f9f 100644
--- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
@@ -19,6 +19,7 @@
 
 <config>
   <jmx />
+  <metrics/>
 
   <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
 
@@ -32,6 +33,9 @@
   </directoryFactory>
   <schemaFactory class="ClassicIndexSchemaFactory"/>
 
+  <indexReaderFactory name="IndexReaderFactory" 
+       class="org.apache.solr.cloud.TrollingIndexReaderFactory"></indexReaderFactory >
+    
   <dataDir>${solr.data.dir:}</dataDir>
 
   <!-- an update processor the explicitly excludes distrib to test
@@ -51,67 +55,32 @@
     </updateLog>
   </updateHandler>
 
-  <updateRequestProcessorChain name="dedupe">
-    <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
-      <bool name="enabled">true</bool>
-      <bool name="overwriteDupes">true</bool>
-      <str name="fields">v_t,t_field</str>
-      <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
-    </processor>
-    <processor class="solr.RunUpdateProcessorFactory" />
-  </updateRequestProcessorChain>
-  <updateRequestProcessorChain name="stored_sig">
-    <!-- this chain is valid even though the signature field is not
-         indexed, because we are not asking for dups to be overwritten
-      -->
-    <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
-      <bool name="enabled">true</bool>
-      <str name="signatureField">non_indexed_signature_sS</str>
-      <bool name="overwriteDupes">false</bool>
-      <str name="fields">v_t,t_field</str>
-      <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
-    </processor>
-    <processor class="solr.RunUpdateProcessorFactory" />
-  </updateRequestProcessorChain>
-
-  <updateRequestProcessorChain name="distrib-dup-test-chain-explicit">
-    <!-- explicit test using processors before and after distrib -->
-    <processor class="solr.RegexReplaceProcessorFactory">
-      <str name="fieldName">regex_dup_A_s</str>
-      <str name="pattern">x</str>
-      <str name="replacement">x_x</str>
-    </processor>
-    <processor class="solr.DistributedUpdateProcessorFactory" />
-    <processor class="solr.RegexReplaceProcessorFactory">
-      <str name="fieldName">regex_dup_B_s</str>
-      <str name="pattern">x</str>
-      <str name="replacement">x_x</str>
-    </processor>
-    <processor class="solr.RunUpdateProcessorFactory" />
-  </updateRequestProcessorChain>
-
-  <updateRequestProcessorChain name="distrib-dup-test-chain-implicit">
-    <!-- implicit test w/o distrib declared-->
-    <processor class="solr.RegexReplaceProcessorFactory">
-      <str name="fieldName">regex_dup_A_s</str>
-      <str name="pattern">x</str>
-      <str name="replacement">x_x</str>
-    </processor>
-    <processor class="solr.RegexReplaceProcessorFactory">
-      <str name="fieldName">regex_dup_B_s</str>
-      <str name="pattern">x</str>
-      <str name="replacement">x_x</str>
-    </processor>
-    <processor class="solr.RunUpdateProcessorFactory" />
-  </updateRequestProcessorChain>
-
+  <query>
+         <filterCache class="solr.FastLRUCache"
+                 size="0"
+                 initialSize="0"
+                 autowarmCount="0"/>
+         <queryResultCache class="solr.LRUCache"
+                  size="0"
+                  initialSize="0"
+                  autowarmCount="0"/>
+         <documentCache class="solr.LRUCache"
+                   size="0"
+                   initialSize="0"
+                   autowarmCount="0"/>
+         <fieldValueCache class="solr.FastLRUCache"
+                size="0"
+                autowarmCount="0"
+                showItems="0" />         
+  </query>
+  
   <searchComponent name="delayingSearchComponent"
                    class="org.apache.solr.search.DelayingSearchComponent"/>
 
   <requestHandler name="/select" class="solr.SearchHandler">
     <arr name="first-components">
       <str>delayingSearchComponent</str>
-    </arr>
+    </arr> 
   </requestHandler>
 
 </config>
diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
index a4dc86ed279..3a45500d11c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
@@ -16,65 +16,100 @@
  */
 package org.apache.solr.cloud;
 
+import java.lang.invoke.MethodHandles;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.MiniSolrCloudCluster.JettySolrRunnerWithMetrics;
+import static org.apache.solr.cloud.TrollingIndexReaderFactory.*;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.handler.component.FacetComponent;
+import org.apache.solr.handler.component.QueryComponent;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.facet.FacetModule;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.codahale.metrics.Metered;
+import com.codahale.metrics.MetricRegistry;
 
 /**
 * Distributed test for {@link org.apache.lucene.index.ExitableDirectoryReader} 
 */
 public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
+  
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   private static final int NUM_DOCS_PER_TYPE = 20;
   private static final String sleep = "2";
 
   private static final String COLLECTION = "exitable";
+  private static Map<String, Metered> fiveHundredsByNode;
   
   @BeforeClass
   public static void setupCluster() throws Exception {
-    configureCluster(2)
-        .addConfig("conf", TEST_PATH().resolve("configsets").resolve("exitable-directory").resolve("conf"))
+    Builder clusterBuilder = configureCluster(2)
+        .addConfig("conf", TEST_PATH().resolve("configsets").resolve("exitable-directory").resolve("conf"));
+    clusterBuilder.withMetrics(true);
+    clusterBuilder
         .configure();
 
     CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1)
         .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
     cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
         (n, c) -> DocCollection.isFullyActive(n, c, 2, 1));
-  }
 
-  @Test
-  public void test() throws Exception {
+    fiveHundredsByNode = new LinkedHashMap<>(); 
+    for (JettySolrRunner jetty: cluster.getJettySolrRunners()) {
+      MetricRegistry metricRegistry = ((JettySolrRunnerWithMetrics)jetty).getMetricRegistry();
+      Metered httpOk = (Metered) metricRegistry.getMetrics()
+          .get("org.eclipse.jetty.servlet.ServletContextHandler.2xx-responses");
+      assertTrue("expeting some http activity during collection creation",httpOk.getCount()>0);
+      
+      Metered old = fiveHundredsByNode.put(jetty.getNodeName(),
+          (Metered) metricRegistry.getMetrics()
+             .get("org.eclipse.jetty.servlet.ServletContextHandler.5xx-responses"));
+      assertNull("expecting uniq nodenames",old);
+    }
+    
     indexDocs();
-    doTimeoutTests();
   }
 
-  public void indexDocs() throws Exception {
-    int counter = 1;
+  public static void indexDocs() throws Exception {
+    int counter;
+    counter = 1;
     UpdateRequest req = new UpdateRequest();
 
     for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ )
-      req.add(sdoc("id", Integer.toString(counter), "name", "a" + counter));
+      req.add(sdoc("id", Integer.toString(counter), "name", "a" + counter,
+          "num",""+counter));
 
     counter++;
     for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ )
-      req.add(sdoc("id", Integer.toString(counter), "name", "b" + counter));
+      req.add(sdoc("id", Integer.toString(counter), "name", "b" + counter,
+          "num",""+counter));
 
     counter++;
     for(; counter % NUM_DOCS_PER_TYPE != 0; counter++ )
-      req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter));
+      req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter,
+          "num",""+counter));
 
     req.commit(cluster.getSolrClient(), COLLECTION);
   }
 
-  public void doTimeoutTests() throws Exception {
+  @Test
+  public void test() throws Exception {
     assertPartialResults(params("q", "name:a*", "timeAllowed", "1", "sleep", sleep));
 
     /*
@@ -99,18 +134,136 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
     assertSuccess(params("q","name:b*")); // no time limitation
   }
 
+  @Test
+  public void testWhitebox() throws Exception {
+    
+    try (Trap catchIds = catchTrace(
+        new CheckMethodName("doProcessSearchByIds"), () -> {})) {
+      assertPartialResults(params("q", "{!cache=false}name:a*", "sort", "query($q,1) asc"),
+          () -> assertTrue(catchIds.hasCaught()));
+    } catch (AssertionError ae) {
+      Trap.dumpLastStackTraces(log);
+      throw ae;
+    }
+
+    // the point is to catch sort_values (fsv) timeout, between search and facet
+    // I haven't find a way to encourage fsv to read index
+    try (Trap catchFSV = catchTrace(
+        new CheckMethodName("doFieldSortValues"), () -> {})) {
+      assertPartialResults(params("q", "{!cache=false}name:a*", "sort", "query($q,1) asc"),
+          () -> assertTrue(catchFSV.hasCaught()));
+    } catch (AssertionError ae) {
+      Trap.dumpLastStackTraces(log);
+      throw ae;
+    }
+    
+    try (Trap catchClass = catchClass(
+        QueryComponent.class.getSimpleName(), () -> {  })) {
+      assertPartialResults(params("q", "{!cache=false}name:a*"),
+          ()->assertTrue(catchClass.hasCaught()));
+    }catch(AssertionError ae) {
+      Trap.dumpLastStackTraces(log);
+      throw ae;
+    }
+    try(Trap catchClass = catchClass(FacetComponent.class.getSimpleName())){
+      assertPartialResults(params("q", "{!cache=false}name:a*", "facet","true", "facet.method", "enum", 
+          "facet.field", "id"),
+          ()->assertTrue(catchClass.hasCaught()));
+    }catch(AssertionError ae) {
+      Trap.dumpLastStackTraces(log);
+      throw ae;
+    }
+
+    try (Trap catchClass = catchClass(FacetModule.class.getSimpleName())) {
+      assertPartialResults(params("q", "{!cache=false}name:a*", "json.facet", "{ ids: {"
+          + " type: range, field : num, start : 0, end : 100, gap : 10 }}"),
+          () -> assertTrue(catchClass.hasCaught()));
+    } catch (AssertionError ae) {
+      Trap.dumpLastStackTraces(log);
+      throw ae;
+    }
+  }
+
+  @Test 
+  @Repeat(iterations=5)
+  public void testCreepThenBite() throws Exception {
+    int creep=100;
+    ModifiableSolrParams params = params("q", "{!cache=false}name:a*");
+    SolrParams cases[] = new SolrParams[] {
+        params( "sort","query($q,1) asc"),
+        params("rows","0", "facet","true", "facet.method", "enum", "facet.field", "name"),
+        params("rows","0", "json.facet","{ ids: { type: range, field : num, start : 1, end : 99, gap : 9 }}")
+        }; //add more cases here 
+
+    params.add(cases[random().nextInt(cases.length)]);
+    for (; ; creep*=1.5) {
+      final int boundary = creep;
+      try(Trap catchClass = catchCount(boundary)){
+        
+        params.set("boundary", boundary);
+        QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, 
+            params);
+        assertEquals(""+rsp, rsp.getStatus(), 0);
+        assertNo500s(""+rsp);
+        if (!isPartial(rsp)) {
+          assertFalse(catchClass.hasCaught());
+          break;
+        }
+        assertTrue(catchClass.hasCaught());
+      }catch(AssertionError ae) {
+        Trap.dumpLastStackTraces(log);
+        throw ae;
+      }
+    }
+    int numBites = atLeast(100);
+    for(int bite=0; bite<numBites; bite++) {
+      int boundary = random().nextInt(creep);
+      try(Trap catchCount = catchCount(boundary)){
+        params.set("boundary", boundary);
+        QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, 
+            params);
+        assertEquals(""+rsp, rsp.getStatus(), 0);
+        assertNo500s(""+rsp);
+        assertEquals(""+creep+" ticks were sucessful; trying "+boundary+" yields "+rsp, 
+            catchCount.hasCaught(), isPartial(rsp));
+      }catch(AssertionError ae) {
+        Trap.dumpLastStackTraces(log);
+        throw ae;
+      }
+    }
+  }
+
+  public boolean isPartial(QueryResponse rsp) {
+    return Boolean.TRUE.equals(rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+  }
+
+  public void assertNo500s(String msg) {
+    assertTrue(msg,fiveHundredsByNode.values().stream().allMatch((m)->m.getCount()==0));
+  }
+  
   /**
    * execute a request, verify that we get an expected error
    */
   public void assertPartialResults(ModifiableSolrParams p) throws Exception {
+    assertPartialResults(p, ()->{});
+  }
+  
+  public void assertPartialResults(ModifiableSolrParams p, Runnable postRequestCheck) throws Exception {
       QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
-      assertEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" were expected",
-          true, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+      postRequestCheck.run();
+      assertEquals(rsp.getStatus(), 0);
+      assertEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" were expected at "+rsp,
+          true, rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+      assertNo500s(""+rsp);
   }
   
   public void assertSuccess(ModifiableSolrParams p) throws Exception {
-    QueryResponse response = cluster.getSolrClient().query(COLLECTION, p);
-    assertEquals("Wrong #docs in response", NUM_DOCS_PER_TYPE - 1, response.getResults().getNumFound());
+    QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
+    assertEquals(rsp.getStatus(), 0);
+    assertEquals("Wrong #docs in response", NUM_DOCS_PER_TYPE - 1, rsp.getResults().getNumFound());
+    assertNotEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" weren't expected "+rsp,
+        true, rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+    assertNo500s(""+rsp);
   }
 }
 
diff --git a/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java b/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java
new file mode 100644
index 00000000000..553ed6f2327
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.ExitableDirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.QueryTimeout;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.StandardIndexReaderFactory;
+
+public class TrollingIndexReaderFactory extends StandardIndexReaderFactory {
+
+  private static volatile Trap trap;
+  private final static BlockingQueue<List<Object>> lastStacktraces = new LinkedBlockingQueue<List<Object>>();
+  private final static long startTime = ManagementFactory.getRuntimeMXBean().getStartTime();
+  private static final int keepStackTraceLines = 20;
+  protected static final int maxTraces = 4;
+
+  
+  private static Trap setTrap(Trap troll) {
+    trap = troll;  
+    return troll;
+  }
+  
+  public static abstract class Trap implements Closeable{
+    protected abstract boolean shouldExit();
+    public abstract boolean hasCaught();
+    @Override
+    public final void close() throws IOException {
+      setTrap(null);
+    }
+    @Override
+    public abstract String toString();
+    
+    public static void dumpLastStackTraces(org.slf4j.Logger log) {
+      ArrayList<List<Object>> stacks = new ArrayList<>();
+      lastStacktraces.drainTo(stacks);
+      StringBuilder out = new StringBuilder("the last caught stacktraces: \n");
+      for(List<Object> stack : stacks) {
+        int l=0;
+        for (Object line : stack) {
+          if (l++>0) {
+            out.append('\t');
+          }
+          out.append(line);
+          out.append('\n');
+        }
+        out.append('\n');
+      }
+      log.error("the last caught traces {}", out);
+    }
+  }
+
+  static final class CheckMethodName implements Predicate<StackTraceElement> {
+    private final String methodName;
+  
+    CheckMethodName(String methodName) {
+      this.methodName = methodName;
+    }
+  
+    @Override
+    public boolean test(StackTraceElement trace) {
+      return trace.getMethodName().equals(methodName);
+    }
+    
+    @Override
+    public String toString() {
+      return "hunting for "+methodName+"()";
+    }
+  }
+
+  public static Trap catchClass(String className) {
+    return catchClass(className, ()->{});
+  }
+  
+  public static Trap catchClass(String className, Runnable onCaught) {
+    Predicate<StackTraceElement> judge = new Predicate<StackTraceElement>() {
+      @Override
+      public boolean test(StackTraceElement trace) {
+        return trace.getClassName().indexOf(className)>=0;
+      }
+      @Override
+      public String toString() {
+        return "className contains "+className;
+      }
+    };
+    return catchTrace(judge, onCaught) ;        
+  }
+  
+  public static Trap catchTrace(Predicate<StackTraceElement> judge, Runnable onCaught) {
+    return setTrap(new Trap() {
+      
+      private boolean trigered;
+
+      @Override
+      protected boolean shouldExit() {
+        Exception e = new Exception("stack sniffer"); 
+        e.fillInStackTrace();
+        StackTraceElement[] stackTrace = e.getStackTrace();
+        for(StackTraceElement trace : stackTrace) {
+          if (judge.test(trace)) {
+            trigered = true; 
+            recordStackTrace(stackTrace);
+            onCaught.run();
+            return true;
+          }
+        }
+        return false;
+      }
+
+      @Override
+      public boolean hasCaught() {
+        return trigered;
+      }
+
+      @Override
+      public String toString() {
+        return ""+judge;
+      }
+    });
+  }
+  
+  public static Trap catchCount(int boundary) {
+    return setTrap(new Trap() {
+      
+      private AtomicInteger count = new AtomicInteger();
+    
+      @Override
+      public String toString() {
+        return ""+count.get()+"th tick of "+boundary+" allowed";
+      }
+      
+      private boolean trigered;
+
+      @Override
+      protected boolean shouldExit() {
+        int now = count.incrementAndGet();
+        boolean trigger = now==boundary 
+            || (now>boundary && LuceneTestCase.rarely(LuceneTestCase.random()));
+        if (trigger) {
+          Exception e = new Exception("stack sniffer"); 
+          e.fillInStackTrace();
+          recordStackTrace(e.getStackTrace());
+          trigered = true;
+        } 
+        return trigger;
+      }
+
+      @Override
+      public boolean hasCaught() {
+        return trigered;
+      }
+    });
+  }
+  
+  private static void recordStackTrace(StackTraceElement[] stackTrace) {
+    //keep the last n limited traces. 
+    //e.printStackTrace();
+    ArrayList<Object> stack = new ArrayList<Object>();
+    stack.add(""+ (new Date().getTime()-startTime)+" ("+Thread.currentThread().getName()+")");
+    for (int l=2; l<stackTrace.length && l<keepStackTraceLines; l++) {
+      stack.add(stackTrace[l]);
+    }
+    lastStacktraces.add(stack);
+    // triming queue 
+    while(lastStacktraces.size()>maxTraces) {
+      try {
+        lastStacktraces.poll(100, TimeUnit.MILLISECONDS);
+      } catch (InterruptedException e1) {
+        e1.printStackTrace();
+      }
+    }
+  }
+
+  @Override
+  public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
+    DirectoryReader newReader = super.newReader(indexDir, core);
+    return wrap(newReader);
+  }
+
+  private ExitableDirectoryReader wrap(DirectoryReader newReader) throws IOException {
+    return new ExitableDirectoryReader(newReader, new QueryTimeout() {
+      @Override
+      public boolean shouldExit() {
+        return trap!=null && trap.shouldExit();
+      }
+      
+      @Override
+      public String toString() {
+        return ""+trap;
+      }
+    });
+  }
+
+  @Override
+  public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+    DirectoryReader newReader = super.newReader(writer, core);
+    return wrap(newReader);
+  }
+}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 2fbeba812c5..deaa4a87ed4 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -16,7 +16,6 @@
  */
 package org.apache.solr.cloud;
 
-import javax.servlet.Filter;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.Charset;
@@ -44,6 +43,8 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import javax.servlet.Filter;
+
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettyConfig;
@@ -71,10 +72,13 @@ import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.util.TimeOut;
 import org.apache.zookeeper.KeeperException;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.MetricRegistry;
+
 /**
  * "Mini" SolrCloud cluster to be used for testing
  */
@@ -124,9 +128,11 @@ public class MiniSolrCloudCluster {
   private final Path baseDir;
   private final CloudSolrClient solrClient;
   private final JettyConfig jettyConfig;
+  private final boolean trackJettyMetrics;
 
   private final AtomicInteger nodeIds = new AtomicInteger();
 
+
   /**
    * Create a MiniSolrCloudCluster with default solr.xml
    *
@@ -230,10 +236,32 @@ public class MiniSolrCloudCluster {
    */
    MiniSolrCloudCluster(int numServers, Path baseDir, String solrXml, JettyConfig jettyConfig,
       ZkTestServer zkTestServer, Optional<String> securityJson) throws Exception {
+     this(numServers, baseDir, solrXml, jettyConfig,
+         zkTestServer,securityJson, false);
+   }
+  /**
+   * Create a MiniSolrCloudCluster.
+   * Note - this constructor visibility is changed to package protected so as to
+   * discourage its usage. Ideally *new* functionality should use {@linkplain SolrCloudTestCase}
+   * to configure any additional parameters.
+   *
+   * @param numServers number of Solr servers to start
+   * @param baseDir base directory that the mini cluster should be run from
+   * @param solrXml solr.xml file to be uploaded to ZooKeeper
+   * @param jettyConfig Jetty configuration
+   * @param zkTestServer ZkTestServer to use.  If null, one will be created
+   * @param securityJson A string representation of security.json file (optional).
+   * @param trackJettyMetrics supply jetties with metrics registry
+   *
+   * @throws Exception if there was an error starting the cluster
+   */
+   MiniSolrCloudCluster(int numServers, Path baseDir, String solrXml, JettyConfig jettyConfig,
+      ZkTestServer zkTestServer, Optional<String> securityJson, boolean trackJettyMetrics) throws Exception {
 
     Objects.requireNonNull(securityJson);
     this.baseDir = Objects.requireNonNull(baseDir);
     this.jettyConfig = Objects.requireNonNull(jettyConfig);
+    this.trackJettyMetrics = trackJettyMetrics;
 
     log.info("Starting cluster of {} servers in {}", numServers, baseDir);
 
@@ -433,12 +461,14 @@ public class MiniSolrCloudCluster {
     Path runnerPath = createInstancePath(name);
     String context = getHostContextSuitableForServletContext(hostContext);
     JettyConfig newConfig = JettyConfig.builder(config).setContext(context).build();
-    JettySolrRunner jetty = new JettySolrRunner(runnerPath.toString(), newConfig);
+    JettySolrRunner jetty = !trackJettyMetrics 
+        ? new JettySolrRunner(runnerPath.toString(), newConfig)
+         :new JettySolrRunnerWithMetrics(runnerPath.toString(), newConfig);
     jetty.start();
     jettys.add(jetty);
     return jetty;
   }
-
+  
   /**
    * Start a new Solr instance, using the default config
    *
@@ -774,4 +804,29 @@ public class MiniSolrCloudCluster {
       throw new TimeoutException("Waiting for Jetty to stop timed out");
     }
   }
+  
+  /** @lucene.experimental */
+  public static final class JettySolrRunnerWithMetrics extends JettySolrRunner {
+    public JettySolrRunnerWithMetrics(String solrHome, JettyConfig config) {
+      super(solrHome, config);
+    }
+
+    private volatile MetricRegistry metricRegistry;
+
+    @Override
+    protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
+      metricRegistry = new MetricRegistry();
+      com.codahale.metrics.jetty9.InstrumentedHandler metrics 
+          = new com.codahale.metrics.jetty9.InstrumentedHandler(
+               metricRegistry);
+      metrics.setHandler(chain);
+      return metrics;
+    }
+
+    /** @return optional subj. It may be null, if it's not yet created. */
+    public MetricRegistry getMetricRegistry() {
+      return metricRegistry;
+    }
+  }
+
 }
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index acb09be5109..647d7b7d23b 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -107,6 +107,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
     private List<Config> configs = new ArrayList<>();
     private Map<String, Object> clusterProperties = new HashMap<>();
 
+    private boolean trackJettyMetrics;
     /**
      * Create a builder
      * @param nodeCount the number of nodes in the cluster
@@ -191,6 +192,10 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
       return this;
     }
 
+    public Builder withMetrics(boolean trackJettyMetrics) {
+      this.trackJettyMetrics = trackJettyMetrics; 
+      return this;
+    }
     /**
      * Configure and run the {@link MiniSolrCloudCluster}
      * @throws Exception if an error occurs on startup
@@ -204,7 +209,8 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
      * @throws Exception if an error occurs on startup
      */
     public MiniSolrCloudCluster build() throws Exception {
-      MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(nodeCount, baseDir, solrxml, jettyConfig, null, securityJson);
+      MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(nodeCount, baseDir, solrxml, jettyConfig,
+          null, securityJson, trackJettyMetrics);
       CloudSolrClient client = cluster.getSolrClient();
       for (Config config : configs) {
         ((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(config.path, config.name);