Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11426: TestLazyCores fails too often. Adding debugging code MASTER ONLY since I can't get it to fail locally

This commit is contained in:
Erick Erickson 2017-10-06 09:54:32 -07:00
parent c5f9a6f221
commit 37fb60d0f1
1 changed files with 86 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
solr/core/src/test/org/apache/solr/core/TestLazyCores.java
View File

@ -84,7 +84,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
private CoreContainer init() throws Exception { private CoreContainer init() throws Exception {
solrHomeDirectory = createTempDir().toFile(); solrHomeDirectory = createTempDir().toFile();
copyXmlToHome(solrHomeDirectory.getAbsoluteFile(), "solr.xml"); copyXmlToHome(solrHomeDirectory.getAbsoluteFile(), "solr.xml");
for (int idx = 1; idx < 10; ++idx) { for (int idx = 1; idx < 10; ++idx) {
copyMinConf(new File(solrHomeDirectory, "collection" + idx)); copyMinConf(new File(solrHomeDirectory, "collection" + idx));
@ -93,7 +93,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
NodeConfig cfg = SolrDispatchFilter.loadNodeConfig(solrHomeDirectory.toPath(), null); NodeConfig cfg = SolrDispatchFilter.loadNodeConfig(solrHomeDirectory.toPath(), null);
return createCoreContainer(cfg, testCores); return createCoreContainer(cfg, testCores);
} }
@Test @Test
public void testLazyLoad() throws Exception { public void testLazyLoad() throws Exception {
CoreContainer cc = init(); CoreContainer cc = init();
@ -172,6 +172,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
, "//result[@numFound='0']" , "//result[@numFound='0']"
); );
} }
@Test @Test
public void testLazySearch() throws Exception { public void testLazySearch() throws Exception {
CoreContainer cc = init(); CoreContainer cc = init();
@ -309,11 +310,12 @@ public class TestLazyCores extends SolrTestCaseJ4 {
//SolrException cause = (SolrException)se.getCause(); //SolrException cause = (SolrException)se.getCause();
assertEquals("Exception code should be 500", 500, se.code()); assertEquals("Exception code should be 500", 500, se.code());
for (String err : errs) { for (String err : errs) {
assertTrue("Should have seen an exception containing the an error", assertTrue("Should have seen an exception containing the an error",
se.getMessage().contains(err)); se.getMessage().contains(err));
} }
} }
} }
@Test @Test
public void testCreateSame() throws Exception { public void testCreateSame() throws Exception {
final CoreContainer cc = init(); final CoreContainer cc = init();
@ -373,7 +375,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
resp); resp);
} }
// Make sure that creating a transient core from the admin handler correctly respects the transient limits etc. // Make sure that creating a transient core from the admin handler correctly respects the transient limits etc.
@Test @Test
public void testCreateTransientFromAdmin() throws Exception { public void testCreateTransientFromAdmin() throws Exception {
@ -406,22 +408,21 @@ public class TestLazyCores extends SolrTestCaseJ4 {
// transient generates a "too many closes" errorl // transient generates a "too many closes" errorl
class TestThread extends Thread { class TestThread extends Thread {
@Override @Override
public void run() { public void run() {
final int sleep_millis = random().nextInt(1000); final int sleep_millis = random().nextInt(1000);
try { try {
if (sleep_millis > 0) { if (sleep_millis > 0) {
if (VERBOSE) { if (VERBOSE) {
System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run sleeping for "+sleep_millis+" ms"); System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run sleeping for " + sleep_millis + " ms");
} }
Thread.sleep(sleep_millis); Thread.sleep(sleep_millis);
} }
} } catch (InterruptedException ie) {
catch (InterruptedException ie) {
if (VERBOSE) { if (VERBOSE) {
System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run caught "+ie+" whilst sleeping for "+sleep_millis+" ms"); System.out.println("TestLazyCores.testCreateTransientFromAdmin Thread.run caught " + ie + " whilst sleeping for " + sleep_millis + " ms");
} }
} }
@ -431,13 +432,14 @@ public class TestLazyCores extends SolrTestCaseJ4 {
c4.close(); c4.close();
c5.close(); c5.close();
} }
}; }
;
// with SOLR-6279 UNLOAD will wait for the core's reference count to have reached zero // with SOLR-6279 UNLOAD will wait for the core's reference count to have reached zero
// hence cN.close() need to proceed or run in parallel with unloadViaAdmin(...) // hence cN.close() need to proceed or run in parallel with unloadViaAdmin(...)
final TestThread cThread = new TestThread(); final TestThread cThread = new TestThread();
cThread.start(); cThread.start();
unloadViaAdmin(cc, "core1"); unloadViaAdmin(cc, "core1");
unloadViaAdmin(cc, "core2"); unloadViaAdmin(cc, "core2");
unloadViaAdmin(cc, "core3"); unloadViaAdmin(cc, "core3");
@ -461,8 +463,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
final CoreContainer cc = initGoodAndBad(Arrays.asList("core1", "core2"), final CoreContainer cc = initGoodAndBad(Arrays.asList("core1", "core2"),
Arrays.asList("badSchema1", "badSchema2"), Arrays.asList("badSchema1", "badSchema2"),
Arrays.asList("badConfig1", "badConfig2")); Arrays.asList("badConfig1", "badConfig2"));
try { try {
// first, did the two good cores load successfully? // first, did the two good cores load successfully?
checkInCores(cc, "core1", "core2"); checkInCores(cc, "core1", "core2");
@ -495,7 +497,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
copyGoodConf("badSchema1", "schema-tiny.xml", "schema.xml"); copyGoodConf("badSchema1", "schema-tiny.xml", "schema.xml");
copyGoodConf("badSchema2", "schema-tiny.xml", "schema.xml"); copyGoodConf("badSchema2", "schema-tiny.xml", "schema.xml");
// Reload the cores and insure that // Reload the cores and insure that
// 1> they pick up the new configs // 1> they pick up the new configs
// 2> they don't fail again b/c they still have entries in loadFailure in core container. // 2> they don't fail again b/c they still have entries in loadFailure in core container.
@ -503,7 +505,8 @@ public class TestLazyCores extends SolrTestCaseJ4 {
cc.reload("badConfig2"); cc.reload("badConfig2");
cc.reload("badSchema1"); cc.reload("badSchema1");
cc.reload("badSchema2"); cc.reload("badSchema2");
SolrCore bc1 = cc.getCore("badConfig1");; SolrCore bc1 = cc.getCore("badConfig1");
;
SolrCore bc2 = cc.getCore("badConfig2"); SolrCore bc2 = cc.getCore("badConfig2");
SolrCore bs1 = cc.getCore("badSchema1"); SolrCore bs1 = cc.getCore("badSchema1");
SolrCore bs2 = cc.getCore("badSchema2"); SolrCore bs2 = cc.getCore("badSchema2");
@ -570,10 +573,10 @@ public class TestLazyCores extends SolrTestCaseJ4 {
List<String> badSchemaCores, List<String> badSchemaCores,
List<String> badConfigCores) throws Exception { List<String> badConfigCores) throws Exception {
solrHomeDirectory = createTempDir().toFile(); solrHomeDirectory = createTempDir().toFile();
// Don't pollute the log with exception traces when they're expected. // Don't pollute the log with exception traces when they're expected.
ignoreException(Pattern.quote("SAXParseException")); ignoreException(Pattern.quote("SAXParseException"));
// Create the cores that should be fine. // Create the cores that should be fine.
for (String coreName : goodCores) { for (String coreName : goodCores) {
File coreRoot = new File(solrHomeDirectory, coreName); File coreRoot = new File(solrHomeDirectory, coreName);
@ -648,18 +651,19 @@ public class TestLazyCores extends SolrTestCaseJ4 {
public static void checkNotInCores(CoreContainer cc, List<String> nameCheck) { public static void checkNotInCores(CoreContainer cc, List<String> nameCheck) {
checkNotInCores(cc, nameCheck, Collections.emptyList()); checkNotInCores(cc, nameCheck, Collections.emptyList());
} }
public static void checkNotInCores(CoreContainer cc, List<String> nameCheck, List<String> namesBad) { public static void checkNotInCores(CoreContainer cc, List<String> nameCheck, List<String> namesBad) {
Collection<String> loadedNames = cc.getLoadedCoreNames(); Collection<String> loadedNames = cc.getLoadedCoreNames();
for (String name : nameCheck) { for (String name : nameCheck) {
assertFalse("core " + name + " was found in the list of cores", loadedNames.contains(name)); assertFalse("core " + name + " was found in the list of cores", loadedNames.contains(name));
} }
// There was a problem at one point exacerbated by the poor naming conventions. So parallel to loaded cores, there // There was a problem at one point exacerbated by the poor naming conventions. So parallel to loaded cores, there
// should be the ability to get the core _names_ that are loaded as well as all the core names _possible_ // should be the ability to get the core _names_ that are loaded as well as all the core names _possible_
// //
// the names above should only contain loaded core names. Every name in names should be in allNames, but none of // the names above should only contain loaded core names. Every name in names should be in allNames, but none of
// the names in nameCheck should be loaded and thus should not be in names. // the names in nameCheck should be loaded and thus should not be in names.
Collection<String> allNames = cc.getAllCoreNames(); Collection<String> allNames = cc.getAllCoreNames();
// Every core that has not failed to load should be in coreDescriptors. // Every core that has not failed to load should be in coreDescriptors.
List<CoreDescriptor> descriptors = cc.getCoreDescriptors(); List<CoreDescriptor> descriptors = cc.getCoreDescriptors();
@ -668,9 +672,9 @@ public class TestLazyCores extends SolrTestCaseJ4 {
for (CoreDescriptor desc : descriptors) { for (CoreDescriptor desc : descriptors) {
assertTrue("Name should have a corresponding descriptor", allNames.contains(desc.getName())); assertTrue("Name should have a corresponding descriptor", allNames.contains(desc.getName()));
} }
// First check that all loaded cores are in allNames. // First check that all loaded cores are in allNames.
for (String name : loadedNames) { for (String name : loadedNames) {
assertTrue("Loaded core " + name + " should have been found in the list of all possible core names", assertTrue("Loaded core " + name + " should have been found in the list of all possible core names",
allNames.contains(name)); allNames.contains(name));
} }
@ -691,10 +695,10 @@ public class TestLazyCores extends SolrTestCaseJ4 {
public static void checkInCores(CoreContainer cc, String... nameCheck) { public static void checkInCores(CoreContainer cc, String... nameCheck) {
Collection<String> loadedNames = cc.getLoadedCoreNames(); Collection<String> loadedNames = cc.getLoadedCoreNames();
assertEquals("There whould be exactly as many loaded cores as loaded names returned. ", assertEquals("There whould be exactly as many loaded cores as loaded names returned. ",
loadedNames.size(), nameCheck.length); loadedNames.size(), nameCheck.length);
for (String name : nameCheck) { for (String name : nameCheck) {
assertTrue("core " + name + " was not found in the list of cores", loadedNames.contains(name)); assertTrue("core " + name + " was not found in the list of cores", loadedNames.contains(name));
} }
@ -703,7 +707,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
private void addLazy(SolrCore core, String... fieldValues) throws IOException { private void addLazy(SolrCore core, String... fieldValues) throws IOException {
UpdateHandler updater = core.getUpdateHandler(); UpdateHandler updater = core.getUpdateHandler();
AddUpdateCommand cmd = new AddUpdateCommand(makeReq(core)); AddUpdateCommand cmd = new AddUpdateCommand(makeReq(core));
cmd.solrDoc = sdoc((Object[])fieldValues); cmd.solrDoc = sdoc((Object[]) fieldValues);
updater.addDoc(cmd); updater.addDoc(cmd);
} }
@ -729,49 +733,49 @@ public class TestLazyCores extends SolrTestCaseJ4 {
@Test @Test
public void testMidUseUnload() throws Exception { public void testMidUseUnload() throws Exception {
final int maximumSleepMillis = random().nextInt(9999) + 1; // sleep for up to 10 s Must add 1 because using final int maximumSleepMillis = random().nextInt(9999) + 1; // sleep for up to 10 s Must add 1 because using
// this as a seed will rea few lines down will // this as a seed will rea few lines down will
// throw an exception if this is zero // throw an exception if this is zero
if (VERBOSE) { if (VERBOSE) {
System.out.println("TestLazyCores.testMidUseUnload maximumSleepMillis="+maximumSleepMillis); System.out.println("TestLazyCores.testMidUseUnload maximumSleepMillis=" + maximumSleepMillis);
} }
class TestThread extends Thread { class TestThread extends Thread {
SolrCore core_to_use = null; SolrCore core_to_use = null;
@Override @Override
public void run() { public void run() {
final int sleep_millis = random().nextInt(maximumSleepMillis); final int sleep_millis = random().nextInt(maximumSleepMillis);
try { try {
if (sleep_millis > 0) { if (sleep_millis > 0) {
if (VERBOSE) { if (VERBOSE) {
System.out.println("TestLazyCores.testMidUseUnload Thread.run sleeping for "+sleep_millis+" ms"); System.out.println("TestLazyCores.testMidUseUnload Thread.run sleeping for " + sleep_millis + " ms");
} }
Thread.sleep(sleep_millis); Thread.sleep(sleep_millis);
} }
} } catch (InterruptedException ie) {
catch (InterruptedException ie) {
if (VERBOSE) { if (VERBOSE) {
System.out.println("TestLazyCores.testMidUseUnload Thread.run caught "+ie+" whilst sleeping for "+sleep_millis+" ms"); System.out.println("TestLazyCores.testMidUseUnload Thread.run caught " + ie + " whilst sleeping for " + sleep_millis + " ms");
} }
} }
assertFalse(core_to_use.isClosed()); // not closed since we are still using it and hold a reference assertFalse(core_to_use.isClosed()); // not closed since we are still using it and hold a reference
core_to_use.close(); // now give up our reference to the core core_to_use.close(); // now give up our reference to the core
} }
}; }
;
CoreContainer cc = init(); CoreContainer cc = init();
try { try {
TestThread thread = new TestThread(); TestThread thread = new TestThread();
thread.core_to_use = cc.getCore("collection1"); thread.core_to_use = cc.getCore("collection1");
assertNotNull(thread.core_to_use); assertNotNull(thread.core_to_use);
assertFalse(thread.core_to_use.isClosed()); // freshly-in-use core is not closed assertFalse(thread.core_to_use.isClosed()); // freshly-in-use core is not closed
thread.start(); thread.start();
unloadViaAdmin(cc, "collection1"); unloadViaAdmin(cc, "collection1");
assertTrue(thread.core_to_use.isClosed()); // after unload-ing the core is closed assertTrue(thread.core_to_use.isClosed()); // after unload-ing the core is closed
@ -804,14 +808,14 @@ public class TestLazyCores extends SolrTestCaseJ4 {
openCores.add(core); openCores.add(core);
add10(core); add10(core);
} }
// Just proving that some cores have been aged out. // Just proving that some cores have been aged out.
checkNotInCores(cc, Arrays.asList("collection2", "collection3")); checkNotInCores(cc, Arrays.asList("collection2", "collection3"));
// Close our get of all cores above. // Close our get of all cores above.
for (SolrCore core : openCores) core.close(); for (SolrCore core : openCores) core.close();
openCores.clear(); openCores.clear();
// We still should have 6, 7, 8, 9 loaded, their reference counts have NOT dropped to zero // We still should have 6, 7, 8, 9 loaded, their reference counts have NOT dropped to zero
checkInCores(cc, "collection1", "collection5", checkInCores(cc, "collection1", "collection5",
"collection6", "collection7", "collection8", "collection9"); "collection6", "collection7", "collection8", "collection9");
@ -820,11 +824,11 @@ public class TestLazyCores extends SolrTestCaseJ4 {
// The point of this test is to insure that when cores are aged out and re-opened // The point of this test is to insure that when cores are aged out and re-opened
// that the docs are there, so insure that the core we're testing is gone, gone, gone. // that the docs are there, so insure that the core we're testing is gone, gone, gone.
checkNotInCores(cc, Arrays.asList(coreName)); checkNotInCores(cc, Arrays.asList(coreName));
// Load the core up again. // Load the core up again.
SolrCore core = cc.getCore(coreName); SolrCore core = cc.getCore(coreName);
openCores.add(core); openCores.add(core);
// Insure docs are still there. // Insure docs are still there.
check10(core); check10(core);
} }
@ -843,7 +847,41 @@ public class TestLazyCores extends SolrTestCaseJ4 {
} }
private void check10(SolrCore core) { private void check10(SolrCore core) {
// Just get a couple of searches to work!
// DEBUGGING ONLY since I can't get this to fail locally (Erick Erickson)
// Wondering if this is the problem with multiple searchers being opened on load, in which case
// looping for a bit should help.
boolean failIt = false;
try {
for (int idx = 0; idx < 10; ++idx) {
LocalSolrQueryRequest lsrg = makeReq(core, "q", "*:*", "wt", "xml");
String resp = h.query(lsrg);
if (resp.contains("numFound=\"10\"")) {
if (failIt) {
fail("***********EOE found 10 docs after failing to find 10 docs the first time, Implicates SOLR-11035 ???");
}
break;
}
failIt = true;
System.out.println("********EOE AT LEAST ONE LOOP FAILED TO FIND 10 DOCS. resp: " + resp);
Thread.sleep(1000);
}
if (failIt) {
System.out.println("********EOE Test failed, what happens if we add a doc now? Will there be 11 docs? ");
addLazy(core, "id", "1000EOE");
SolrQueryRequest req = makeReq(core);
CommitUpdateCommand cmtCmd = new CommitUpdateCommand(req, false);
core.getUpdateHandler().commit(cmtCmd);
assertQ("test closing core without committing after adding 11th document",
makeReq(core, "q", "*:*")
, "//result[@numFound='11']"
);
fail("*************EOE added 11th doc and then found 11! Implicates SOLR-11035 ??? ");
}
} catch (Exception e) {
fail(e.getMessage());
}
// Just get a simple search to work!
assertQ("test closing core without committing", assertQ("test closing core without committing",
makeReq(core, "q", "*:*") makeReq(core, "q", "*:*")
, "//result[@numFound='10']" , "//result[@numFound='10']"