diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java index b0aaaae01af..97342d1e0dd 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java @@ -78,11 +78,11 @@ public class TestRecovery extends SolrTestCaseJ4 { assertU(commit()); Deque versions = new ArrayDeque(); - versions.addFirst(addAndGetVersion(sdoc("id", "1"), null)); - versions.addFirst(addAndGetVersion(sdoc("id", "11"), null)); - versions.addFirst(addAndGetVersion(sdoc("id", "12"), null)); - versions.addFirst(deleteByQueryAndGetVersion("id:11", null)); - versions.addFirst(addAndGetVersion(sdoc("id", "13"), null)); + versions.addFirst(addAndGetVersion(sdoc("id", "A1"), null)); + versions.addFirst(addAndGetVersion(sdoc("id", "A11"), null)); + versions.addFirst(addAndGetVersion(sdoc("id", "A12"), null)); + versions.addFirst(deleteByQueryAndGetVersion("id:A11", null)); + versions.addFirst(addAndGetVersion(sdoc("id", "A13"), null)); assertJQ(req("q","*:*"),"/response/numFound==0"); @@ -114,10 +114,10 @@ public class TestRecovery extends SolrTestCaseJ4 { // make sure we can still access versions after recovery assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions); - assertU(adoc("id","2")); - assertU(adoc("id","3")); - assertU(delI("2")); - assertU(adoc("id","4")); + assertU(adoc("id","A2")); + assertU(adoc("id","A3")); + assertU(delI("A2")); + assertU(adoc("id","A4")); assertJQ(req("q","*:*") ,"/response/numFound==3"); @@ -129,7 +129,7 @@ public class TestRecovery extends SolrTestCaseJ4 { // wait until recovery has finished assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS)); assertJQ(req("q","*:*") ,"/response/numFound==5"); - assertJQ(req("q","id:2") ,"/response/numFound==0"); + assertJQ(req("q","id:A2") ,"/response/numFound==0"); // no updates, so insure that recovery does not run h.close(); @@ -197,12 +197,12 @@ public class TestRecovery extends SolrTestCaseJ4 { assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); // simulate updates from a leader - updateJ(jsonAdd(sdoc("id","1", "_version_","1010")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","11", "_version_","1015")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonDelQ("id:1 id:11 id:2 id:3"), params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-1017")); - updateJ(jsonAdd(sdoc("id","2", "_version_","1020")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","3", "_version_","1030")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - deleteAndGetVersion("1", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2010")); + updateJ(jsonAdd(sdoc("id","B1", "_version_","1010")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B11", "_version_","1015")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-1017")); + updateJ(jsonAdd(sdoc("id","B2", "_version_","1020")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B3", "_version_","1030")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + deleteAndGetVersion("B1", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2010")); assertJQ(req("qt","/get", "getVersions","6") ,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}" @@ -222,7 +222,7 @@ public class TestRecovery extends SolrTestCaseJ4 { // real-time get should also not show anything (this could change in the future, // but it's currently used for validating version numbers too, so it would // be bad for updates to be visible if we're just buffering. - assertJQ(req("qt","/get", "id","3") + assertJQ(req("qt","/get", "id","B3") ,"=={'doc':null}" ); @@ -251,22 +251,22 @@ public class TestRecovery extends SolrTestCaseJ4 { ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); - Long ver = getVer(req("qt","/get", "id","3")); + Long ver = getVer(req("qt","/get", "id","B3")); assertEquals(1030L, ver.longValue()); // add a reordered doc that shouldn't overwrite one in the index - updateJ(jsonAdd(sdoc("id","3", "_version_","3")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B3", "_version_","3")), params(SEEN_LEADER,SEEN_LEADER_VAL)); // reorder two buffered updates - updateJ(jsonAdd(sdoc("id","4", "_version_","1040")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - deleteAndGetVersion("4", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-940")); // this update should not take affect - updateJ(jsonAdd(sdoc("id","6", "_version_","1060")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","5", "_version_","1050")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","8", "_version_","1080")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B4", "_version_","1040")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + deleteAndGetVersion("B4", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-940")); // this update should not take affect + updateJ(jsonAdd(sdoc("id","B6", "_version_","1060")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B5", "_version_","1050")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B8", "_version_","1080")), params(SEEN_LEADER,SEEN_LEADER_VAL)); // test that delete by query is at least buffered along with everything else so it will delete the // currently buffered id:8 (even if it doesn't currently support versioning) - updateJ("{\"delete\": { \"query\":\"id:2 OR id:8\" }}", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-3000")); + updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-3000")); assertJQ(req("qt","/get", "getVersions","13") ,"=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the "3" appears because versions aren't checked while buffering @@ -281,22 +281,22 @@ public class TestRecovery extends SolrTestCaseJ4 { logReplay.release(1); // now add another update - updateJ(jsonAdd(sdoc("id","7", "_version_","1070")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","B7", "_version_","1070")), params(SEEN_LEADER,SEEN_LEADER_VAL)); // a reordered update that should be dropped - deleteAndGetVersion("5", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-950")); + deleteAndGetVersion("B5", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-950")); - deleteAndGetVersion("6", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2060")); + deleteAndGetVersion("B6", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2060")); logReplay.release(1000); UpdateLog.RecoveryInfo recInfo = rinfoFuture.get(); assertJQ(req("q", "*:*", "sort","id asc", "fl","id,_version_") , "/response/docs==[" - + "{'id':'3','_version_':1030}" - + ",{'id':'4','_version_':1040}" - + ",{'id':'5','_version_':1050}" - + ",{'id':'7','_version_':1070}" + + "{'id':'B3','_version_':1030}" + + ",{'id':'B4','_version_':1040}" + + ",{'id':'B5','_version_':1050}" + + ",{'id':'B7','_version_':1070}" +"]" ); @@ -359,14 +359,14 @@ public class TestRecovery extends SolrTestCaseJ4 { assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); // simulate updates from a leader - updateJ(jsonAdd(sdoc("id","1", "_version_","101")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","2", "_version_","102")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","3", "_version_","103")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C1", "_version_","101")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C2", "_version_","102")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C3", "_version_","103")), params(SEEN_LEADER,SEEN_LEADER_VAL)); assertTrue(ulog.dropBufferedUpdates()); ulog.bufferUpdates(); - updateJ(jsonAdd(sdoc("id", "4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id", "5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id", "C4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id", "C5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); logReplay.release(1000); rinfoFuture = ulog.applyBufferedUpdates(); @@ -378,17 +378,17 @@ public class TestRecovery extends SolrTestCaseJ4 { ); // this time add some docs first before buffering starts (so tlog won't be at pos 0) - updateJ(jsonAdd(sdoc("id","100", "_version_","200")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","101", "_version_","201")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C100", "_version_","200")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C101", "_version_","201")), params(SEEN_LEADER,SEEN_LEADER_VAL)); ulog.bufferUpdates(); - updateJ(jsonAdd(sdoc("id","103", "_version_","203")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","104", "_version_","204")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C103", "_version_","203")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C104", "_version_","204")), params(SEEN_LEADER,SEEN_LEADER_VAL)); assertTrue(ulog.dropBufferedUpdates()); ulog.bufferUpdates(); - updateJ(jsonAdd(sdoc("id","105", "_version_","205")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","106", "_version_","206")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C105", "_version_","205")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","C106", "_version_","206")), params(SEEN_LEADER,SEEN_LEADER_VAL)); rinfoFuture = ulog.applyBufferedUpdates(); rinfo = rinfoFuture.get(); @@ -396,12 +396,12 @@ public class TestRecovery extends SolrTestCaseJ4 { assertJQ(req("q", "*:*", "sort","_version_ asc", "fl","id,_version_") , "/response/docs==[" - + "{'id':'4','_version_':104}" - + ",{'id':'5','_version_':105}" - + ",{'id':'100','_version_':200}" - + ",{'id':'101','_version_':201}" - + ",{'id':'105','_version_':205}" - + ",{'id':'106','_version_':206}" + + "{'id':'C4','_version_':104}" + + ",{'id':'C5','_version_':105}" + + ",{'id':'C100','_version_':200}" + + ",{'id':'C101','_version_':201}" + + ",{'id':'C105','_version_':205}" + + ",{'id':'C106','_version_':206}" +"]" ); @@ -428,18 +428,18 @@ public class TestRecovery extends SolrTestCaseJ4 { clearIndex(); assertU(commit()); - assertU(adoc("id","1", "val_i","1")); - assertU(adoc("id","2", "val_i","1")); + assertU(adoc("id","D1", "val_i","1")); + assertU(adoc("id","D2", "val_i","1")); assertU(commit()); - long v1 = getVer(req("q","id:1")); - long v1a = getVer(req("q","id:2")); + long v1 = getVer(req("q","id:D1")); + long v1a = getVer(req("q","id:D2")); h.close(); createCore(); - assertU(adoc("id","1", "val_i","2")); + assertU(adoc("id","D1", "val_i","2")); assertU(commit()); - long v2 = getVer(req("q","id:1")); + long v2 = getVer(req("q","id:D1")); assert(v2 > v1); @@ -483,8 +483,8 @@ public class TestRecovery extends SolrTestCaseJ4 { clearIndex(); assertU(commit()); - assertU(adoc("id","1", "val_i","1")); - assertU(adoc("id","2", "val_i","1")); + assertU(adoc("id","E1", "val_i","1")); + assertU(adoc("id","E2", "val_i","1")); // set to a high enough number so this test won't hang on a bug logReplay.release(10); @@ -658,9 +658,9 @@ public class TestRecovery extends SolrTestCaseJ4 { clearIndex(); assertU(commit()); - assertU(adoc("id","1")); - assertU(adoc("id","2")); - assertU(adoc("id","3")); + assertU(adoc("id","F1")); + assertU(adoc("id","F2")); + assertU(adoc("id","F3")); h.close(); String[] files = UpdateLog.getLogList(logDir); @@ -683,9 +683,9 @@ public class TestRecovery extends SolrTestCaseJ4 { // Now test that the bad log file doesn't mess up retrieving latest versions // - updateJ(jsonAdd(sdoc("id","4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); - updateJ(jsonAdd(sdoc("id","6", "_version_","106")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","F4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","F5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); + updateJ(jsonAdd(sdoc("id","F6", "_version_","106")), params(SEEN_LEADER,SEEN_LEADER_VAL)); // This currently skips the bad log file and also returns the version of the clearIndex (del *:*) // assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]"); @@ -698,7 +698,20 @@ public class TestRecovery extends SolrTestCaseJ4 { } } + // stops the core, removes the transaction logs, restarts the core. + void deleteLogs() throws Exception { + File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir(); + h.close(); + + String[] files = UpdateLog.getLogList(logDir); + for (String file : files) { + new File(logDir, file).delete(); + } + + assertEquals(0, UpdateLog.getLogList(logDir).length); + createCore(); + } private static Long getVer(SolrQueryRequest req) throws Exception { String response = JQ(req);