SOLR-10898: Fix SOLR-10898 to not deterministicly fail 1/512 runs

solr/CHANGES.txt

@@ -482,6 +482,8 @@ Other Changes
 * SOLR-11021: The elevate.xml config-file is made optional in the ElevationComponent.
   The default configset doesn't ship with a elevate.xml file anymore (Varun Thacker)
 
+* SOLR-10898: Fix SOLR-10898 to not deterministicly fail 1/512 runs (hossman)
+
 ==================  6.7.0 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java

@@ -19,9 +19,12 @@ package org.apache.solr.cloud;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+
 
 import com.codahale.metrics.Counter;
 import org.apache.lucene.util.TestUtil;
@@ -41,7 +44,6 @@ import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.request.SolrRequestHandler;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -86,6 +88,25 @@ public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase
 
     cloudClient.getZkStateReader().forceUpdateCollection("b1x1");
 
+    // get direct access to the metrics counters for each core/replica we're interested to monitor them
+    final Map<String,Counter> counters = new LinkedHashMap<>();
+    for (JettySolrRunner runner : jettys) {
+      CoreContainer container = runner.getCoreContainer();
+      SolrMetricManager metricManager = container.getMetricManager();
+      for (SolrCore core : container.getCores()) {
+        if ("a1x2".equals(core.getCoreDescriptor().getCollectionName())) {
+          String registry = core.getCoreMetricManager().getRegistryName();
+          Counter cnt = metricManager.counter(null, registry, "requests", "QUERY./select");
+          // sanity check
+          assertEquals(core.getName() + " has already recieved some requests?",
+                       0, cnt.getCount());
+          counters.put(core.getName(), cnt);
+        }
+      }
+    }
+    assertEquals("Sanity Check: we know there should be 2 replicas", 2, counters.size());
+
+    // send queries to the node that doesn't host any core/replica and see where it routes them
     ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
     DocCollection b1x1 = clusterState.getCollection("b1x1");
     Collection<Replica> replicas = b1x1.getSlice("shard1").getReplicas();
@@ -94,29 +115,30 @@ public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase
     if (!baseUrl.endsWith("/")) baseUrl += "/";
     try (HttpSolrClient client = getHttpSolrClient(baseUrl + "a1x2", 2000, 5000)) {
 
+      long expectedTotalRequests = 0;
+      Set<String> uniqueCoreNames = new LinkedHashSet<>();
+      
       log.info("Making requests to " + baseUrl + "a1x2");
-      for (int i = 0; i < 10; i++) {
+      while (uniqueCoreNames.size() < counters.keySet().size() && expectedTotalRequests < 1000L) {
+        expectedTotalRequests++;
         client.query(new SolrQuery("*:*"));
+
+        long actualTotalRequests = 0;
+        for (Map.Entry<String,Counter> e : counters.entrySet()) {
+          final long coreCount = e.getValue().getCount();
+          actualTotalRequests += coreCount;
+          if (0 < coreCount) {
+            uniqueCoreNames.add(e.getKey());
+          }
+        }
+        assertEquals("Sanity Check: Num Queries So Far Doesn't Match Total????",
+                     expectedTotalRequests, actualTotalRequests);
       }
-    }
+      log.info("Total requests: " + expectedTotalRequests);
-
+      assertEquals("either request randomization code is broken of this test seed is really unlucky, " +
-    Map<String, Integer> shardVsCount = new HashMap<>();
+                   "Gave up waiting for requests to hit every core at least once after " +
-    for (JettySolrRunner runner : jettys) {
+                   expectedTotalRequests + " requests",
-      CoreContainer container = runner.getCoreContainer();
+                   uniqueCoreNames.size(), counters.size());
-      SolrMetricManager metricManager = container.getMetricManager();
-      for (SolrCore core : container.getCores()) {
-        String registry = core.getCoreMetricManager().getRegistryName();
-        Counter cnt = metricManager.counter(null, registry, "requests", "QUERY./select");
-        SolrRequestHandler select = core.getRequestHandler("");
-//        long c = (long) select.getStatistics().get("requests");
-        shardVsCount.put(core.getName(), (int) cnt.getCount());
-      }
-    }
-
-    log.info("Shard count map = " + shardVsCount);
-
-    for (Map.Entry<String, Integer> entry : shardVsCount.entrySet()) {
-      assertTrue("Shard " + entry.getKey() + " received all 10 requests", entry.getValue() != 10);
     }
   }