From 42f13731b3a037ee9682df49bb946ca0b4ca8544 Mon Sep 17 00:00:00 2001 From: markrmiller Date: Wed, 12 Dec 2018 18:21:06 -0600 Subject: [PATCH] SOLR-12801: Harden SimSolrCloudTests. --- .../autoscaling/sim/SimSolrCloudTestCase.java | 39 ------------------- .../sim/TestSimComputePlanAction.java | 10 +---- .../sim/TestSimExecutePlanAction.java | 13 ++++--- .../sim/TestSimExtremeIndexing.java | 18 ++++++--- .../autoscaling/sim/TestSimLargeCluster.java | 23 ++++++----- .../sim/TestSimNodeAddedTrigger.java | 16 ++++---- .../sim/TestSimNodeLostTrigger.java | 15 +++---- .../autoscaling/sim/TestSimPolicyCloud.java | 16 +++++--- .../sim/TestSimTriggerIntegration.java | 24 +++--------- 9 files changed, 66 insertions(+), 108 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java index 69954cd80d6..611a46f867c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java @@ -16,13 +16,10 @@ */ package org.apache.solr.cloud.autoscaling.sim; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; @@ -34,14 +31,10 @@ import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; -import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,38 +81,6 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 { super.setUp(); } - @Before - public void checkClusterConfiguration() throws Exception { - if (cluster == null) - throw new RuntimeException("SimCloudManager not configured - have you called configureCluster()?"); - // clear any persisted configuration - cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1); - cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1); - cluster.getSimNodeStateProvider().simRemoveDeadNodes(); - cluster.getSimClusterStateProvider().simRemoveDeadNodes(); - // restore the expected number of nodes - int currentSize = cluster.getLiveNodesSet().size(); - if (currentSize < clusterNodeCount) { - int addCnt = clusterNodeCount - currentSize; - while (addCnt-- > 0) { - cluster.simAddNode(); - } - } else if (currentSize > clusterNodeCount) { - cluster.simRemoveRandomNodes(currentSize - clusterNodeCount, true, random()); - } - // clean any persisted trigger state or events - removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - removeChildren(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - cluster.getSimClusterStateProvider().simResetLeaderThrottles(); - cluster.simRestartOverseer(null); - cluster.getSimClusterStateProvider().simDeleteAllCollections(); - cluster.simClearSystemCollection(); - cluster.getTimeSource().sleep(10000); - cluster.simResetOpCounts(); - } - protected void removeChildren(String path) throws Exception { TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java index 255f80056e9..efaa558da02 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java @@ -52,7 +52,6 @@ import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.LogLevel; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,14 +72,9 @@ public class TestSimComputePlanAction extends SimSolrCloudTestCase { private static final AtomicReference actionContextPropsRef = new AtomicReference<>(); private static final AtomicReference eventRef = new AtomicReference<>(); - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1, TimeSource.get("simTime:50")); - } - @Before public void init() throws Exception { - + configureCluster(1, TimeSource.get("simTime:50")); fired.set(false); triggerFiredLatch = new CountDownLatch(1); actionContextPropsRef.set(null); @@ -118,7 +112,7 @@ public class TestSimComputePlanAction extends SimSolrCloudTestCase { for (String coll: cluster.getSimClusterStateProvider().simListCollections()) { log.info("* Collection " + coll + " state: " + state.getCollection(coll)); } - + shutdownCluster(); } @Test diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java index a9c84be55c2..07b02c9f6c0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import com.google.common.collect.Lists; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -44,15 +43,17 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.Utils; import org.apache.solr.util.LogLevel; -import org.apache.solr.common.util.TimeSource; import org.junit.After; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; + /** * Test for {@link ExecutePlanAction} */ @@ -62,8 +63,8 @@ public class TestSimExecutePlanAction extends SimSolrCloudTestCase { private static final int NODE_COUNT = 2; - @BeforeClass - public static void setupCluster() throws Exception { + @Before + public void setupCluster() throws Exception { configureCluster(NODE_COUNT, TimeSource.get("simTime:50")); } @@ -76,7 +77,7 @@ public class TestSimExecutePlanAction extends SimSolrCloudTestCase { for (String coll: cluster.getSimClusterStateProvider().simListCollections()) { log.info("* Collection " + coll + " state: " + state.getCollection(coll)); } - + shutdownCluster(); } @Test diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java index 15d676b523b..c648e545844 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java @@ -16,13 +16,13 @@ */ package org.apache.solr.cloud.autoscaling.sim; +import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest; + import java.lang.invoke.MethodHandles; import java.util.Iterator; import java.util.Locale; import java.util.concurrent.TimeUnit; -import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; - import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -37,13 +37,14 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.TimeSource; import org.apache.solr.util.LogLevel; +import org.junit.After; import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; /** * @@ -75,8 +76,8 @@ public class TestSimExtremeIndexing extends SimSolrCloudTestCase { private static TimeSource timeSource; private static SolrClient solrClient; - @BeforeClass - public static void setupCluster() throws Exception { + @Before + public void setupCluster() throws Exception { configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); timeSource = cluster.getTimeSource(); solrClient = cluster.simGetSolrClient(); @@ -87,6 +88,11 @@ public class TestSimExtremeIndexing extends SimSolrCloudTestCase { public static void tearDownCluster() throws Exception { solrClient = null; } + + @After + public void afterTest() throws Exception { + shutdownCluster(); + } @Test public void testScaleUp() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java index c8527e111a8..8298831fbc8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java @@ -62,7 +62,6 @@ import org.apache.solr.util.LogLevel; import org.apache.solr.util.TimeOut; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +73,7 @@ import org.slf4j.LoggerFactory; public class TestSimLargeCluster extends SimSolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final int SPEED = 50; + public static final int SPEED = 100; public static final int NUM_NODES = 100; @@ -85,19 +84,15 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase { static CountDownLatch triggerFinishedLatch; static int waitForSeconds; - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); - } - @After public void tearDownTest() throws Exception { shutdownCluster(); - configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); } @Before public void setupTest() throws Exception { + configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); + waitForSeconds = 5; triggerStartedCount.set(0); triggerFinishedCount.set(0); @@ -351,11 +346,16 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase { log.info("Ready after " + CloudTestUtils.waitForState(cluster, collectionName, 20 * NUM_NODES, TimeUnit.SECONDS, CloudTestUtils.clusterShape(NUM_NODES / 10, NUM_NODES / 8 * 3, false, true)) + " ms"); - int count = 50; + int count = 1000; SolrInputDocument finishedEvent = null; long lastNumOps = cluster.simGetOpCount("MOVEREPLICA"); while (count-- > 0) { cluster.getTimeSource().sleep(10000); + + if (cluster.simGetOpCount("MOVEREPLICA") < 2) { + continue; + } + long currentNumOps = cluster.simGetOpCount("MOVEREPLICA"); if (currentNumOps == lastNumOps) { int size = systemColl.size() - 1; @@ -432,7 +432,6 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase { if (cluster != null) { cluster.close(); } - setupCluster(); setUp(); setupTest(); long total = doTestNodeLost(wait, delay * 1000, 0); @@ -699,9 +698,9 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase { // wait for listener to capture the SUCCEEDED stage cluster.getTimeSource().sleep(25000); - assertNotNull(listenerEvents.entrySet().toString(), listenerEvents.get("srt")); + assertNotNull(listenerEvents.entrySet().iterator().toString(), listenerEvents.get("srt")); - assertTrue(listenerEvents.entrySet().toString(), listenerEvents.get("srt").size() >= 1); + assertTrue(listenerEvents.entrySet().iterator().toString(), listenerEvents.get("srt").size() >= 1); CapturedEvent ev = listenerEvents.get("srt").get(0); assertEquals(TriggerEventType.SEARCHRATE, ev.event.getEventType()); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java index c9e506cb008..928046a8822 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java @@ -36,8 +36,8 @@ import org.apache.solr.cloud.autoscaling.TriggerEvent; import org.apache.solr.cloud.autoscaling.TriggerValidationException; import org.apache.solr.common.util.TimeSource; import org.apache.solr.core.SolrResourceLoader; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; /** @@ -60,18 +60,20 @@ public class TestSimNodeAddedTrigger extends SimSolrCloudTestCase { private static TimeSource timeSource; - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1, TimeSource.get("simTime:" + SPEED)); - timeSource = cluster.getTimeSource(); - } - @Before public void beforeTest() throws Exception { + configureCluster(1, TimeSource.get("simTime:" + SPEED)); + timeSource = cluster.getTimeSource(); + actionConstructorCalled = new AtomicBoolean(false); actionInitCalled = new AtomicBoolean(false); actionCloseCalled = new AtomicBoolean(false); } + + @After + public void afterTest() throws Exception { + shutdownCluster(); + } @Test public void testTrigger() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java index 0a9bd19d718..8eb6156b936 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java @@ -37,8 +37,8 @@ import org.apache.solr.cloud.autoscaling.TriggerEvent; import org.apache.solr.cloud.autoscaling.TriggerValidationException; import org.apache.solr.common.util.TimeSource; import org.apache.solr.core.SolrResourceLoader; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; /** @@ -60,18 +60,19 @@ public class TestSimNodeLostTrigger extends SimSolrCloudTestCase { // currentTimeMillis is not as precise so to avoid false positives while comparing time of fire, we add some delta private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(5, TimeSource.get("simTime:" + SPEED)); - timeSource = cluster.getTimeSource(); - } - @Before public void beforeTest() throws Exception { + configureCluster(5, TimeSource.get("simTime:" + SPEED)); + timeSource = cluster.getTimeSource(); actionConstructorCalled = new AtomicBoolean(false); actionInitCalled = new AtomicBoolean(false); actionCloseCalled = new AtomicBoolean(false); } + + @After + public void afterTest() throws Exception { + shutdownCluster(); + } @Test public void testTrigger() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java index eb251010db0..c8be2322091 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java @@ -16,6 +16,8 @@ */ package org.apache.solr.cloud.autoscaling.sim; +import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -47,13 +49,12 @@ import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.Utils; import org.apache.solr.util.LogLevel; import org.apache.zookeeper.KeeperException; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.rules.ExpectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest; - @LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") public class TestSimPolicyCloud extends SimSolrCloudTestCase { @@ -61,10 +62,15 @@ public class TestSimPolicyCloud extends SimSolrCloudTestCase { @org.junit.Rule public ExpectedException expectedException = ExpectedException.none(); - @BeforeClass - public static void setupCluster() throws Exception { + @Before + public void setupCluster() throws Exception { configureCluster(5, TimeSource.get("simTime:50")); } + + @After + public void afterTest() throws Exception { + shutdownCluster(); + } public void testDataProviderPerReplicaDetails() throws Exception { SolrClient solrClient = cluster.simGetSolrClient(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java index c8b65283da8..7e24f4ff5e5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java @@ -68,9 +68,8 @@ import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.LogLevel; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,14 +102,9 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase { private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2, TimeSource.get("simTime:" + SPEED)); - } - - @AfterClass - public static void teardownCluster() { - cluster.simClearSystemCollection(); + @After + public void afterTest() throws Exception { + shutdownCluster(); } private static CountDownLatch getTriggerFiredLatch() { @@ -131,6 +125,8 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase { @Before public void setupTest() throws Exception { + configureCluster(2, TimeSource.get("simTime:" + SPEED)); + // disable .scheduled_maintenance String suspendTriggerCommand = "{" + "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" + @@ -157,14 +153,6 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase { triggerFinishedCount = new AtomicInteger(); events.clear(); listenerEvents.clear(); - cluster.getLiveNodesSet().removeAllLiveNodesListeners(); - while (cluster.getClusterStateProvider().getLiveNodes().size() < 2) { - // perhaps a test stopped a node but didn't start it back - // lets start a node - cluster.simAddNode(); - cluster.getTimeSource().sleep(1000); - } - cluster.getTimeSource().sleep(10000); } @Test