CloudExitableDirectoryReaderTest improvements

remove use of LbSolrClient to prevent premature failure of low timeAllowed options on slow jenkins machines

increase cluster size to also test codepaths where requests are proxied by a node that does not host any core in the collection

solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java

@@ -25,6 +25,7 @@ import com.carrotsearch.randomizedtesting.annotations.Repeat;
 import com.codahale.metrics.Metered;
 import com.codahale.metrics.MetricRegistry;
 import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -37,6 +38,7 @@ import org.apache.solr.handler.component.FacetComponent;
 import org.apache.solr.handler.component.QueryComponent;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.facet.FacetModule;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -60,35 +62,63 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
 
   private static final String COLLECTION = "exitable";
   private static Map<String, Metered> fiveHundredsByNode;
+
+  /**
+   * Client used for all test requests.
+   * <p>
+   * LBSolrClient (and by extension CloudSolrClient) has it's own enforcement of timeAllowed 
+   * in an attempt to prevent "retrying" failed requests far longer then the client requested.
+   * Because of this client side logic, we do not want to use any LBSolrClient (derivative) in 
+   * this test, in order to ensure that on a "slow" machine, the client doesn't pre-emptively 
+   * abort any of our requests that use very low 'timeAllowed' values.
+   * </p>
+   * <p>
+   * ie: This test is not about testing the SolrClient, so keep the SOlrClient simple.
+   * </p>
+   */
+  private static SolrClient client;
   
   @BeforeClass
   public static void setupCluster() throws Exception {
-    Builder clusterBuilder = configureCluster(2)
+    // create one more node then shard, so that we also test the case of proxied requests.
+    Builder clusterBuilder = configureCluster(3)
         .addConfig("conf", TEST_PATH().resolve("configsets").resolve("exitable-directory").resolve("conf"));
     clusterBuilder.withMetrics(true);
     clusterBuilder
         .configure();
 
+    // pick an arbitrary node to use for our requests
+    client = cluster.getRandomJetty(random()).newClient();
+
     CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1)
         .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
     cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
         (n, c) -> DocCollection.isFullyActive(n, c, 2, 1));
 
-    fiveHundredsByNode = new LinkedHashMap<>(); 
+    fiveHundredsByNode = new LinkedHashMap<>();
+    int httpOk = 0;
     for (JettySolrRunner jetty: cluster.getJettySolrRunners()) {
       MetricRegistry metricRegistry = ((JettySolrRunnerWithMetrics)jetty).getMetricRegistry();
-      Metered httpOk = (Metered) metricRegistry.getMetrics()
-          .get("org.eclipse.jetty.servlet.ServletContextHandler.2xx-responses");
-      assertTrue("expeting some http activity during collection creation",httpOk.getCount()>0);
+      
+      httpOk += ((Metered) metricRegistry.getMetrics()
+                 .get("org.eclipse.jetty.servlet.ServletContextHandler.2xx-responses")).getCount();
       
       Metered old = fiveHundredsByNode.put(jetty.getNodeName(),
           (Metered) metricRegistry.getMetrics()
              .get("org.eclipse.jetty.servlet.ServletContextHandler.5xx-responses"));
       assertNull("expecting uniq nodenames",old);
     }
-    
+    assertTrue("expecting some http activity during collection creation", httpOk > 0);
     indexDocs();
   }
+  
+  @AfterClass
+  public static void closeClient() throws Exception {
+    if (null != client) {
+      client.close();
+      client = null;
+    }
+  }
 
   public static void indexDocs() throws Exception {
     int counter;
@@ -109,7 +139,7 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
       req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter,
           "num",""+counter));
 
-    req.commit(cluster.getSolrClient(), COLLECTION);
+    req.commit(client, COLLECTION);
   }
 
   @Test
@@ -205,7 +235,7 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
       try(Trap catchClass = catchCount(boundary)){
         
         params.set("boundary", boundary);
-        QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, 
+        QueryResponse rsp = client.query(COLLECTION, 
             params);
         assertEquals(""+rsp, rsp.getStatus(), 0);
         assertNo500s(""+rsp);
@@ -226,7 +256,7 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
       try(Trap catchCount = catchCount(boundary)){
         params.set("omitHeader", "" + omitHeader);
         params.set("boundary", boundary);
-        QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, 
+        QueryResponse rsp = client.query(COLLECTION, 
             params);
         assertEquals(""+rsp, rsp.getStatus(), 0);
         assertNo500s(""+rsp);
@@ -260,7 +290,7 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
   }
   
   public void assertPartialResults(ModifiableSolrParams p, Runnable postRequestCheck) throws Exception {
-      QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
+      QueryResponse rsp = client.query(COLLECTION, p);
       postRequestCheck.run();
       assertEquals(rsp.getStatus(), 0);
       assertEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" were expected at "+rsp,
@@ -269,7 +299,7 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
   }
   
   public void assertSuccess(ModifiableSolrParams p) throws Exception {
-    QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
+    QueryResponse rsp = client.query(COLLECTION, p);
     assertEquals(rsp.getStatus(), 0);
     assertEquals("Wrong #docs in response", NUM_DOCS_PER_TYPE - 1, rsp.getResults().getNumFound());
     assertNotEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" weren't expected "+rsp,