Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-3126: test changes to handle deletes surviving restart 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1291003 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2012-02-19 14:13:45 +00:00
parent e2ebd116a1
commit 2e07171d1f
1 changed files with 76 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
solr/core/src/test/org/apache/solr/search/TestRecovery.java
View File

@ -78,11 +78,11 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertU(commit()); assertU(commit());
Deque<Long> versions = new ArrayDeque<Long>(); Deque<Long> versions = new ArrayDeque<Long>();
versions.addFirst(addAndGetVersion(sdoc("id", "1"), null)); versions.addFirst(addAndGetVersion(sdoc("id", "A1"), null));
versions.addFirst(addAndGetVersion(sdoc("id", "11"), null)); versions.addFirst(addAndGetVersion(sdoc("id", "A11"), null));
versions.addFirst(addAndGetVersion(sdoc("id", "12"), null)); versions.addFirst(addAndGetVersion(sdoc("id", "A12"), null));
versions.addFirst(deleteByQueryAndGetVersion("id:11", null)); versions.addFirst(deleteByQueryAndGetVersion("id:A11", null));
versions.addFirst(addAndGetVersion(sdoc("id", "13"), null)); versions.addFirst(addAndGetVersion(sdoc("id", "A13"), null));
assertJQ(req("q","*:*"),"/response/numFound==0"); assertJQ(req("q","*:*"),"/response/numFound==0");
@ -114,10 +114,10 @@ public class TestRecovery extends SolrTestCaseJ4 {
// make sure we can still access versions after recovery // make sure we can still access versions after recovery
assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions); assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
assertU(adoc("id","2")); assertU(adoc("id","A2"));
assertU(adoc("id","3")); assertU(adoc("id","A3"));
assertU(delI("2")); assertU(delI("A2"));
assertU(adoc("id","4")); assertU(adoc("id","A4"));
assertJQ(req("q","*:*") ,"/response/numFound==3"); assertJQ(req("q","*:*") ,"/response/numFound==3");
@ -129,7 +129,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
// wait until recovery has finished // wait until recovery has finished
assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS)); assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
assertJQ(req("q","*:*") ,"/response/numFound==5"); assertJQ(req("q","*:*") ,"/response/numFound==5");
assertJQ(req("q","id:2") ,"/response/numFound==0"); assertJQ(req("q","id:A2") ,"/response/numFound==0");
// no updates, so insure that recovery does not run // no updates, so insure that recovery does not run
h.close(); h.close();
@ -197,12 +197,12 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader // simulate updates from a leader
updateJ(jsonAdd(sdoc("id","1", "_version_","1010")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B1", "_version_","1010")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","11", "_version_","1015")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B11", "_version_","1015")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonDelQ("id:1 id:11 id:2 id:3"), params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-1017")); updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-1017"));
updateJ(jsonAdd(sdoc("id","2", "_version_","1020")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B2", "_version_","1020")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","3", "_version_","1030")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B3", "_version_","1030")), params(SEEN_LEADER,SEEN_LEADER_VAL));
deleteAndGetVersion("1", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2010")); deleteAndGetVersion("B1", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2010"));
assertJQ(req("qt","/get", "getVersions","6") assertJQ(req("qt","/get", "getVersions","6")
,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}" ,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}"
@ -222,7 +222,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
// real-time get should also not show anything (this could change in the future, // real-time get should also not show anything (this could change in the future,
// but it's currently used for validating version numbers too, so it would // but it's currently used for validating version numbers too, so it would
// be bad for updates to be visible if we're just buffering. // be bad for updates to be visible if we're just buffering.
assertJQ(req("qt","/get", "id","3") assertJQ(req("qt","/get", "id","B3")
,"=={'doc':null}" ,"=={'doc':null}"
); );
@ -251,22 +251,22 @@ public class TestRecovery extends SolrTestCaseJ4 {
ulog.bufferUpdates(); ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Long ver = getVer(req("qt","/get", "id","3")); Long ver = getVer(req("qt","/get", "id","B3"));
assertEquals(1030L, ver.longValue()); assertEquals(1030L, ver.longValue());
// add a reordered doc that shouldn't overwrite one in the index // add a reordered doc that shouldn't overwrite one in the index
updateJ(jsonAdd(sdoc("id","3", "_version_","3")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B3", "_version_","3")), params(SEEN_LEADER,SEEN_LEADER_VAL));
// reorder two buffered updates // reorder two buffered updates
updateJ(jsonAdd(sdoc("id","4", "_version_","1040")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B4", "_version_","1040")), params(SEEN_LEADER,SEEN_LEADER_VAL));
deleteAndGetVersion("4", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-940")); // this update should not take affect deleteAndGetVersion("B4", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-940")); // this update should not take affect
updateJ(jsonAdd(sdoc("id","6", "_version_","1060")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B6", "_version_","1060")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","5", "_version_","1050")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B5", "_version_","1050")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","8", "_version_","1080")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B8", "_version_","1080")), params(SEEN_LEADER,SEEN_LEADER_VAL));
// test that delete by query is at least buffered along with everything else so it will delete the // test that delete by query is at least buffered along with everything else so it will delete the
// currently buffered id:8 (even if it doesn't currently support versioning) // currently buffered id:8 (even if it doesn't currently support versioning)
updateJ("{\"delete\": { \"query\":\"id:2 OR id:8\" }}", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-3000")); updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-3000"));
assertJQ(req("qt","/get", "getVersions","13") assertJQ(req("qt","/get", "getVersions","13")
,"=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the "3" appears because versions aren't checked while buffering ,"=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the "3" appears because versions aren't checked while buffering
@ -281,22 +281,22 @@ public class TestRecovery extends SolrTestCaseJ4 {
logReplay.release(1); logReplay.release(1);
// now add another update // now add another update
updateJ(jsonAdd(sdoc("id","7", "_version_","1070")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","B7", "_version_","1070")), params(SEEN_LEADER,SEEN_LEADER_VAL));
// a reordered update that should be dropped // a reordered update that should be dropped
deleteAndGetVersion("5", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-950")); deleteAndGetVersion("B5", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-950"));
deleteAndGetVersion("6", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2060")); deleteAndGetVersion("B6", params(SEEN_LEADER,SEEN_LEADER_VAL, "_version_","-2060"));
logReplay.release(1000); logReplay.release(1000);
UpdateLog.RecoveryInfo recInfo = rinfoFuture.get(); UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
assertJQ(req("q", "*:*", "sort","id asc", "fl","id,_version_") assertJQ(req("q", "*:*", "sort","id asc", "fl","id,_version_")
, "/response/docs==[" , "/response/docs==["
+ "{'id':'3','_version_':1030}" + "{'id':'B3','_version_':1030}"
+ ",{'id':'4','_version_':1040}" + ",{'id':'B4','_version_':1040}"
+ ",{'id':'5','_version_':1050}" + ",{'id':'B5','_version_':1050}"
+ ",{'id':'7','_version_':1070}" + ",{'id':'B7','_version_':1070}"
+"]" +"]"
); );
@ -359,14 +359,14 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader // simulate updates from a leader
updateJ(jsonAdd(sdoc("id","1", "_version_","101")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C1", "_version_","101")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","2", "_version_","102")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C2", "_version_","102")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","3", "_version_","103")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C3", "_version_","103")), params(SEEN_LEADER,SEEN_LEADER_VAL));
assertTrue(ulog.dropBufferedUpdates()); assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates(); ulog.bufferUpdates();
updateJ(jsonAdd(sdoc("id", "4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id", "C4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id", "5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id", "C5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL));
logReplay.release(1000); logReplay.release(1000);
rinfoFuture = ulog.applyBufferedUpdates(); rinfoFuture = ulog.applyBufferedUpdates();
@ -378,17 +378,17 @@ public class TestRecovery extends SolrTestCaseJ4 {
); );
// this time add some docs first before buffering starts (so tlog won't be at pos 0) // this time add some docs first before buffering starts (so tlog won't be at pos 0)
updateJ(jsonAdd(sdoc("id","100", "_version_","200")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C100", "_version_","200")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","101", "_version_","201")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C101", "_version_","201")), params(SEEN_LEADER,SEEN_LEADER_VAL));
ulog.bufferUpdates(); ulog.bufferUpdates();
updateJ(jsonAdd(sdoc("id","103", "_version_","203")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C103", "_version_","203")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","104", "_version_","204")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C104", "_version_","204")), params(SEEN_LEADER,SEEN_LEADER_VAL));
assertTrue(ulog.dropBufferedUpdates()); assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates(); ulog.bufferUpdates();
updateJ(jsonAdd(sdoc("id","105", "_version_","205")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C105", "_version_","205")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","106", "_version_","206")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","C106", "_version_","206")), params(SEEN_LEADER,SEEN_LEADER_VAL));
rinfoFuture = ulog.applyBufferedUpdates(); rinfoFuture = ulog.applyBufferedUpdates();
rinfo = rinfoFuture.get(); rinfo = rinfoFuture.get();
@ -396,12 +396,12 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertJQ(req("q", "*:*", "sort","_version_ asc", "fl","id,_version_") assertJQ(req("q", "*:*", "sort","_version_ asc", "fl","id,_version_")
, "/response/docs==[" , "/response/docs==["
+ "{'id':'4','_version_':104}" + "{'id':'C4','_version_':104}"
+ ",{'id':'5','_version_':105}" + ",{'id':'C5','_version_':105}"
+ ",{'id':'100','_version_':200}" + ",{'id':'C100','_version_':200}"
+ ",{'id':'101','_version_':201}" + ",{'id':'C101','_version_':201}"
+ ",{'id':'105','_version_':205}" + ",{'id':'C105','_version_':205}"
+ ",{'id':'106','_version_':206}" + ",{'id':'C106','_version_':206}"
+"]" +"]"
); );
@ -428,18 +428,18 @@ public class TestRecovery extends SolrTestCaseJ4 {
clearIndex(); clearIndex();
assertU(commit()); assertU(commit());
assertU(adoc("id","1", "val_i","1")); assertU(adoc("id","D1", "val_i","1"));
assertU(adoc("id","2", "val_i","1")); assertU(adoc("id","D2", "val_i","1"));
assertU(commit()); assertU(commit());
long v1 = getVer(req("q","id:1")); long v1 = getVer(req("q","id:D1"));
long v1a = getVer(req("q","id:2")); long v1a = getVer(req("q","id:D2"));
h.close(); h.close();
createCore(); createCore();
assertU(adoc("id","1", "val_i","2")); assertU(adoc("id","D1", "val_i","2"));
assertU(commit()); assertU(commit());
long v2 = getVer(req("q","id:1")); long v2 = getVer(req("q","id:D1"));
assert(v2 > v1); assert(v2 > v1);
@ -483,8 +483,8 @@ public class TestRecovery extends SolrTestCaseJ4 {
clearIndex(); clearIndex();
assertU(commit()); assertU(commit());
assertU(adoc("id","1", "val_i","1")); assertU(adoc("id","E1", "val_i","1"));
assertU(adoc("id","2", "val_i","1")); assertU(adoc("id","E2", "val_i","1"));
// set to a high enough number so this test won't hang on a bug // set to a high enough number so this test won't hang on a bug
logReplay.release(10); logReplay.release(10);
@ -658,9 +658,9 @@ public class TestRecovery extends SolrTestCaseJ4 {
clearIndex(); clearIndex();
assertU(commit()); assertU(commit());
assertU(adoc("id","1")); assertU(adoc("id","F1"));
assertU(adoc("id","2")); assertU(adoc("id","F2"));
assertU(adoc("id","3")); assertU(adoc("id","F3"));
h.close(); h.close();
String[] files = UpdateLog.getLogList(logDir); String[] files = UpdateLog.getLogList(logDir);
@ -683,9 +683,9 @@ public class TestRecovery extends SolrTestCaseJ4 {
// Now test that the bad log file doesn't mess up retrieving latest versions // Now test that the bad log file doesn't mess up retrieving latest versions
// //
updateJ(jsonAdd(sdoc("id","4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","F4", "_version_","104")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","F5", "_version_","105")), params(SEEN_LEADER,SEEN_LEADER_VAL));
updateJ(jsonAdd(sdoc("id","6", "_version_","106")), params(SEEN_LEADER,SEEN_LEADER_VAL)); updateJ(jsonAdd(sdoc("id","F6", "_version_","106")), params(SEEN_LEADER,SEEN_LEADER_VAL));
// This currently skips the bad log file and also returns the version of the clearIndex (del *:*) // This currently skips the bad log file and also returns the version of the clearIndex (del *:*)
// assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]"); // assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
@ -698,7 +698,20 @@ public class TestRecovery extends SolrTestCaseJ4 {
} }
} }
// stops the core, removes the transaction logs, restarts the core.
void deleteLogs() throws Exception {
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
h.close();
String[] files = UpdateLog.getLogList(logDir);
for (String file : files) {
new File(logDir, file).delete();
}
assertEquals(0, UpdateLog.getLogList(logDir).length);
createCore();
}
private static Long getVer(SolrQueryRequest req) throws Exception { private static Long getVer(SolrQueryRequest req) throws Exception {
String response = JQ(req); String response = JQ(req);