From d5181ec8e5bbc8932f606caaf281ff6913a1537d Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Mon, 13 Mar 2017 16:55:50 +0530 Subject: [PATCH] SOLR-10079: Increasing threadpool termination to 15s, clearIndex() to do clean replica indexes as well --- .../update/TestInPlaceUpdatesDistrib.java | 89 ++++++++++++++----- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java index 4c90bc65559..bb0ab9adeea 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java @@ -151,15 +151,18 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { "docValues",Boolean.TRUE)); // Do the tests now: - reorderedDBQIndividualReplicaTest(); - testDBQUsingUpdatedFieldFromDroppedUpdate(); - outOfOrderDBQsTest(); docValuesUpdateTest(); ensureRtgWorksWithPartialUpdatesTest(); - delayedReorderingFetchesMissingUpdateFromLeaderTest(); outOfOrderUpdatesIndividualReplicaTest(); - outOfOrderDeleteUpdatesIndividualReplicaTest(); - reorderedDBQsWithInPlaceUpdatesShouldNotThrowReplicaInLIRTest(); + delayedReorderingFetchesMissingUpdateFromLeaderTest(); + updatingDVsInAVeryOldSegment(); + + // TODO Should we combine all/some of these into a single test, so as to cut down on execution time? + reorderedDBQIndividualReplicaTest(); + reorderedDeletesTest(); + reorderedDBQsSimpleTest(); + reorderedDBQsResurrectionTest(); + reorderedDBQsUsingUpdatedValueFromADroppedUpdate(); } private void mapReplicasToClients() throws KeeperException, InterruptedException { @@ -195,7 +198,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { final int NUM_RETRIES = 100, WAIT_TIME = 10; // The following should work: full update to doc 0, in-place update for doc 0, delete doc 0 - private void outOfOrderDBQsTest() throws Exception { + private void reorderedDBQsSimpleTest() throws Exception { clearIndex(); commit(); @@ -243,7 +246,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(10, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); // assert all requests were successful for (Future resp: updateResponses) { @@ -256,7 +259,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { assertNull("This doc was supposed to have been deleted, but was: " + doc, doc); } - log.info("outOfOrderDeleteUpdatesIndividualReplicaTest: This test passed fine..."); + log.info("reorderedDBQsSimpleTest: This test passed fine..."); clearIndex(); commit(); } @@ -294,7 +297,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(10, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); // assert all requests were successful for (Future resp: updateResponses) { @@ -390,6 +393,36 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { log.info("docValuesUpdateTest: This test passed fine..."); } + /** + * Ingest many documents, keep committing. Then update a document from a very old segment. + */ + private void updatingDVsInAVeryOldSegment() throws Exception { + clearIndex(); + commit(); + + String id = String.valueOf(Integer.MAX_VALUE); + index("id", id, "inplace_updatable_float", "1", "title_s", "newtitle"); + + // create 10 more segments + for (int i=0; i<10; i++) { + buildRandomIndex(101.0F, Collections.emptyList()); + } + + index("id", id, "inplace_updatable_float", map("inc", "1")); + + for (SolrClient client: new SolrClient[] {LEADER, NONLEADERS.get(0), NONLEADERS.get(1)}) { + assertEquals("newtitle", client.getById(id).get("title_s")); + assertEquals(2.0f, client.getById(id).get("inplace_updatable_float")); + } + commit(); + for (SolrClient client: new SolrClient[] {LEADER, NONLEADERS.get(0), NONLEADERS.get(1)}) { + assertEquals("newtitle", client.getById(id).get("title_s")); + assertEquals(2.0f, client.getById(id).get("inplace_updatable_float")); + } + + log.info("updatingDVsInAVeryOldSegment: This test passed fine..."); + } + /** * Retries the specified 'req' against each SolrClient in "clients" untill the expected number of * results are returned, at which point the results are verified using assertDocIdsAndValuesInResults @@ -610,7 +643,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(10, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); // assert all requests were successful for (Future resp: updateResponses) { @@ -633,7 +666,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } // The following should work: full update to doc 0, in-place update for doc 0, delete doc 0 - private void outOfOrderDeleteUpdatesIndividualReplicaTest() throws Exception { + private void reorderedDeletesTest() throws Exception { clearIndex(); commit(); @@ -680,7 +713,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(10, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); // assert all requests were successful for (Future resp: updateResponses) { @@ -693,7 +726,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { assertNull("This doc was supposed to have been deleted, but was: " + doc, doc); } - log.info("outOfOrderDeleteUpdatesIndividualReplicaTest: This test passed fine..."); + log.info("reorderedDeletesTest: This test passed fine..."); clearIndex(); commit(); } @@ -707,7 +740,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { DBQ(q=val:10, v=4) DV(id=x, val=5, ver=3) */ - private void reorderedDBQsWithInPlaceUpdatesShouldNotThrowReplicaInLIRTest() throws Exception { + private void reorderedDBQsResurrectionTest() throws Exception { clearIndex(); commit(); @@ -754,7 +787,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(10, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); int successful = 0; for (Future resp: updateResponses) { @@ -794,7 +827,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { assertEquals("Client: "+((HttpSolrClient)client).getBaseURL(), 5, doc.getFieldValue(field)); } - log.info("reorderedDBQsWithInPlaceUpdatesShouldNotThrowReplicaInLIRTest: This test passed fine..."); + log.info("reorderedDBQsResurrectionTest: This test passed fine..."); clearIndex(); commit(); } @@ -829,7 +862,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } threadpool.shutdown(); - assertTrue("Thread pool didn't terminate within 10 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); + assertTrue("Thread pool didn't terminate within 15 secs", threadpool.awaitTermination(15, TimeUnit.SECONDS)); commit(); @@ -1104,7 +1137,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { * inp(id=1,inpfield=14,prevVersion=2,version=3) // will wait till timeout, and then fetch a "not found" from leader * dbq("inp:14",version=4) */ - private void testDBQUsingUpdatedFieldFromDroppedUpdate() throws Exception { + private void reorderedDBQsUsingUpdatedValueFromADroppedUpdate() throws Exception { clearIndex(); commit(); @@ -1161,7 +1194,21 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { assertNull(client.getById("1", params("distrib", "false"))); } - log.info("testDBQUsingUpdatedFieldFromDroppedUpdate: This test passed fine..."); + log.info("reorderedDBQsUsingUpdatedValueFromADroppedUpdate: This test passed fine..."); + } + + @Override + public void clearIndex() { + super.clearIndex(); + try { + for (SolrClient client: new SolrClient[] {LEADER, NONLEADERS.get(0), NONLEADERS.get(1)}) { + if (client != null) { + client.request(simulatedDeleteRequest("*:*", -Long.MAX_VALUE)); + client.commit(); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } } - }