SOLR-11426: TestLazyCores fails too often. Adding debugging code MASTER ONLY since I can't get it to fail locally

solr/core/src/test/org/apache/solr/core/TestLazyCores.java

@@ -172,6 +172,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
         , "//result[@numFound='0']"
     );
   }
+
   @Test
   public void testLazySearch() throws Exception {
     CoreContainer cc = init();
@@ -309,11 +310,12 @@ public class TestLazyCores extends SolrTestCaseJ4 {
       //SolrException cause = (SolrException)se.getCause();
       assertEquals("Exception code should be 500", 500, se.code());
       for (String err : errs) {
-       assertTrue("Should have seen an exception containing the an error",
+        assertTrue("Should have seen an exception containing the an error",
             se.getMessage().contains(err));
       }
     }
   }
+
   @Test
   public void testCreateSame() throws Exception {
     final CoreContainer cc = init();
@@ -414,14 +416,13 @@ public class TestLazyCores extends SolrTestCaseJ4 {
           try {
             if (sleep_millis > 0) {
               if (VERBOSE) {
-                System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run sleeping for "+sleep_millis+" ms");
+                System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run sleeping for " + sleep_millis + " ms");
               }
               Thread.sleep(sleep_millis);
             }
-          }
+          } catch (InterruptedException ie) {
-          catch (InterruptedException ie) {
             if (VERBOSE) {
-              System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run caught "+ie+" whilst sleeping for "+sleep_millis+" ms");
+              System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run caught " + ie + " whilst sleeping for " + sleep_millis + " ms");
             }
           }
 
@@ -431,7 +432,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
           c4.close();
           c5.close();
         }
-      };
+      }
+      ;
 
       // with SOLR-6279 UNLOAD will wait for the core's reference count to have reached zero
       // hence cN.close() need to proceed or run in parallel with unloadViaAdmin(...)
@@ -503,7 +505,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
       cc.reload("badConfig2");
       cc.reload("badSchema1");
       cc.reload("badSchema2");
-      SolrCore bc1 = cc.getCore("badConfig1");;
+      SolrCore bc1 = cc.getCore("badConfig1");
+      ;
       SolrCore bc2 = cc.getCore("badConfig2");
       SolrCore bs1 = cc.getCore("badSchema1");
       SolrCore bs2 = cc.getCore("badSchema2");
@@ -648,6 +651,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
   public static void checkNotInCores(CoreContainer cc, List<String> nameCheck) {
     checkNotInCores(cc, nameCheck, Collections.emptyList());
   }
+
   public static void checkNotInCores(CoreContainer cc, List<String> nameCheck, List<String> namesBad) {
     Collection<String> loadedNames = cc.getLoadedCoreNames();
     for (String name : nameCheck) {
@@ -703,7 +707,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
   private void addLazy(SolrCore core, String... fieldValues) throws IOException {
     UpdateHandler updater = core.getUpdateHandler();
     AddUpdateCommand cmd = new AddUpdateCommand(makeReq(core));
-    cmd.solrDoc = sdoc((Object[])fieldValues);
+    cmd.solrDoc = sdoc((Object[]) fieldValues);
     updater.addDoc(cmd);
   }
 
@@ -729,10 +733,10 @@ public class TestLazyCores extends SolrTestCaseJ4 {
   @Test
   public void testMidUseUnload() throws Exception {
     final int maximumSleepMillis = random().nextInt(9999) + 1; // sleep for up to 10 s Must add 1 because using
-                                                               // this as a seed will rea few lines down will
+    // this as a seed will rea few lines down will
-                                                               // throw an exception if this is zero
+    // throw an exception if this is zero
     if (VERBOSE) {
-      System.out.println("TestLazyCores.testMidUseUnload maximumSleepMillis="+maximumSleepMillis);
+      System.out.println("TestLazyCores.testMidUseUnload maximumSleepMillis=" + maximumSleepMillis);
     }
 
     class TestThread extends Thread {
@@ -746,21 +750,21 @@ public class TestLazyCores extends SolrTestCaseJ4 {
         try {
           if (sleep_millis > 0) {
             if (VERBOSE) {
-              System.out.println("TestLazyCores.testMidUseUnload Thread.run sleeping for "+sleep_millis+" ms");
+              System.out.println("TestLazyCores.testMidUseUnload Thread.run sleeping for " + sleep_millis + " ms");
             }
             Thread.sleep(sleep_millis);
           }
-        }
+        } catch (InterruptedException ie) {
-        catch (InterruptedException ie) {
           if (VERBOSE) {
-            System.out.println("TestLazyCores.testMidUseUnload Thread.run caught "+ie+" whilst sleeping for "+sleep_millis+" ms");
+            System.out.println("TestLazyCores.testMidUseUnload Thread.run caught " + ie + " whilst sleeping for " + sleep_millis + " ms");
           }
         }
 
         assertFalse(core_to_use.isClosed()); // not closed since we are still using it and hold a reference
         core_to_use.close(); // now give up our reference to the core
       }
-    };
+    }
+    ;
 
     CoreContainer cc = init();
 
@@ -843,7 +847,41 @@ public class TestLazyCores extends SolrTestCaseJ4 {
   }
 
   private void check10(SolrCore core) {
-    // Just get a couple of searches to work!
+
+    // DEBUGGING ONLY since I can't get this to fail locally (Erick Erickson)
+    // Wondering if this is the problem with multiple searchers being opened on load, in which case
+    // looping for a bit should help.
+    boolean failIt = false;
+    try {
+      for (int idx = 0; idx < 10; ++idx) {
+        LocalSolrQueryRequest lsrg = makeReq(core, "q", "*:*", "wt", "xml");
+        String resp = h.query(lsrg);
+        if (resp.contains("numFound=\"10\"")) {
+          if (failIt) {
+            fail("***********EOE found 10 docs after failing to find 10 docs the first time, Implicates SOLR-11035 ???");
+          }
+          break;
+        }
+        failIt = true;
+        System.out.println("********EOE AT LEAST ONE LOOP FAILED TO FIND 10 DOCS. resp: " + resp);
+        Thread.sleep(1000);
+      }
+      if (failIt) {
+        System.out.println("********EOE Test failed, what happens if we add a doc now? Will there be 11 docs? ");
+        addLazy(core, "id", "1000EOE");
+        SolrQueryRequest req = makeReq(core);
+        CommitUpdateCommand cmtCmd = new CommitUpdateCommand(req, false);
+        core.getUpdateHandler().commit(cmtCmd);
+        assertQ("test closing core without committing after adding 11th document",
+            makeReq(core, "q", "*:*")
+            , "//result[@numFound='11']"
+        );
+        fail("*************EOE added 11th doc and then found 11! Implicates SOLR-11035 ??? ");
+      }
+    } catch (Exception e) {
+      fail(e.getMessage());
+    }
+    // Just get a simple search to work!
     assertQ("test closing core without committing",
         makeReq(core, "q", "*:*")
         , "//result[@numFound='10']"