From 83c0c952b644bfa89d36a0ec0f1bbd7b34ee0b4e Mon Sep 17 00:00:00 2001 From: Ramkumar Aiyengar Date: Wed, 25 Mar 2015 00:03:33 +0000 Subject: [PATCH] SOLR-7291: Test indexing on ZK disconnect with ChaosMonkey tests git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1669026 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 4 +- .../cloud/ChaosMonkeyNothingIsSafeTest.java | 62 +++++++------ .../solr/cloud/ChaosMonkeySafeLeaderTest.java | 10 +-- .../org/apache/solr/cloud/RecoveryZkTest.java | 10 +-- .../HdfsWriteToMultipleCollectionsTest.java | 8 +- .../solr/BaseDistributedSearchTestCase.java | 6 +- .../solr/cloud/AbstractDistribZkTestBase.java | 9 ++ .../cloud/AbstractFullDistribZkTestBase.java | 86 +++---------------- ...read.java => StoppableIndexingThread.java} | 8 +- .../solr/cloud/StoppableSearchThread.java | 82 ++++++++++++++++++ 10 files changed, 164 insertions(+), 121 deletions(-) rename solr/test-framework/src/java/org/apache/solr/cloud/{StopableIndexingThread.java => StoppableIndexingThread.java} (93%) create mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/StoppableSearchThread.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 838fd50a3c0..1ad2b74132e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -387,12 +387,14 @@ Other Changes * SOLR-6673: MDC based logging of collection, shard, replica, core (Ishan Chattopadhyaya , Noble Paul) +* SOLR-7291: Test indexing on ZK disconnect with ChaosMonkey tests (Ramkumar Aiyengar) + ================== 5.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. NOTE: Solr 5.0 only supports creating and removing SolrCloud collections through - the collections API, unlike previous versions. While not using the + the collections API, unlike previous versions. While not using the collections API may still work in 5.0, it is unsupported, not recommended, and the behavior will change in a 5.x release. diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 91f985f8a2f..2ce851ea37b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -115,11 +115,15 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; } fixShardCount(numShards); + + // None of the operations used here are particularly costly, so this should work. + // Using this low timeout will also help us catch index stalling. + clientSoTimeout = 5000; } @Test public void test() throws Exception { - boolean testsSuccesful = false; + boolean testSuccessful = false; try { handle.clear(); handle.put("timestamp", SKIPVAL); @@ -133,14 +137,14 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase // we cannot do delete by query // as it's not supported for recovery - del("*:*"); + del("*:*"); - List threads = new ArrayList<>(); - List indexTreads = new ArrayList<>(); + List threads = new ArrayList<>(); + List indexTreads = new ArrayList<>(); int threadCount = TEST_NIGHTLY ? 3 : 1; int i = 0; for (i = 0; i < threadCount; i++) { - StopableIndexingThread indexThread = new StopableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); + StoppableIndexingThread indexThread = new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); threads.add(indexThread); indexTreads.add(indexThread); indexThread.start(); @@ -149,7 +153,7 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase threadCount = 1; i = 0; for (i = 0; i < threadCount; i++) { - StopableSearchThread searchThread = new StopableSearchThread(); + StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient); threads.add(searchThread); searchThread.start(); } @@ -158,7 +162,7 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer boolean runFullThrottle = random().nextBoolean(); if (runFullThrottle) { - FullThrottleStopableIndexingThread ftIndexThread = new FullThrottleStopableIndexingThread( + FullThrottleStoppableIndexingThread ftIndexThread = new FullThrottleStoppableIndexingThread( clients, "ft1", true); threads.add(ftIndexThread); ftIndexThread.start(); @@ -184,15 +188,18 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase } finally { chaosMonkey.stopTheMonkey(); } - - for (StopableThread indexThread : threads) { + + // ideally this should go into chaosMonkey + restartZk(1000 * (5 + random().nextInt(4))); + + for (StoppableThread indexThread : threads) { indexThread.safeStop(); } // start any downed jetties to be sure we still will end up with a leader per shard... // wait for stop... - for (StopableThread indexThread : threads) { + for (StoppableThread indexThread : threads) { indexThread.join(); } @@ -217,9 +224,11 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase // we expect full throttle fails, but cloud client should not easily fail - for (StopableThread indexThread : threads) { - if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) { - assertFalse("There were too many update fails - we expect it can happen, but shouldn't easily", ((StopableIndexingThread) indexThread).getFailCount() > FAIL_TOLERANCE); + for (StoppableThread indexThread : threads) { + if (indexThread instanceof StoppableIndexingThread && !(indexThread instanceof FullThrottleStoppableIndexingThread)) { + int failCount = ((StoppableIndexingThread) indexThread).getFailCount(); + assertFalse("There were too many update fails (" + failCount + " > " + FAIL_TOLERANCE + + ") - we expect it can happen, but shouldn't easily", failCount > FAIL_TOLERANCE); } } @@ -247,11 +256,8 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase // sometimes we restart zookeeper as well if (random().nextBoolean()) { - zkServer.shutdown(); - zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); - zkServer.run(); + restartZk(1000 * (5 + random().nextInt(4))); } - try (CloudSolrClient client = createCloudClient("collection1")) { createCollection(null, "testcollection", @@ -263,31 +269,31 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase numShardsNumReplicas.add(1); checkForCollection("testcollection", numShardsNumReplicas, null); - testsSuccesful = true; + testSuccessful = true; } finally { - if (!testsSuccesful) { + if (!testSuccessful) { printLayout(); } } } - private Set getAddFails(List threads) { + private Set getAddFails(List threads) { Set addFails = new HashSet(); - for (StopableIndexingThread thread : threads) { + for (StoppableIndexingThread thread : threads) { addFails.addAll(thread.getAddFails()); } return addFails; } - private Set getDeleteFails(List threads) { + private Set getDeleteFails(List threads) { Set deleteFails = new HashSet(); - for (StopableIndexingThread thread : threads) { + for (StoppableIndexingThread thread : threads) { deleteFails.addAll(thread.getDeleteFails()); } return deleteFails; } - class FullThrottleStopableIndexingThread extends StopableIndexingThread { + class FullThrottleStoppableIndexingThread extends StoppableIndexingThread { private CloseableHttpClient httpClient = HttpClientUtil.createClient(null); private volatile boolean stop = false; int clientIndex = 0; @@ -295,14 +301,14 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase private List clients; private AtomicInteger fails = new AtomicInteger(); - public FullThrottleStopableIndexingThread(List clients, - String id, boolean doDeletes) { + public FullThrottleStoppableIndexingThread(List clients, + String id, boolean doDeletes) { super(controlClient, cloudClient, id, doDeletes); setName("FullThrottleStopableIndexingThread"); setDaemon(true); this.clients = clients; - HttpClientUtil.setConnectionTimeout(httpClient, 15000); - HttpClientUtil.setSoTimeout(httpClient, 15000); + HttpClientUtil.setConnectionTimeout(httpClient, clientConnectionTimeout); + HttpClientUtil.setSoTimeout(httpClient, clientSoTimeout); cusc = new ConcurrentUpdateSolrClient( ((HttpSolrClient) clients.get(0)).getBaseURL(), httpClient, 8, 2) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index d9e835709b6..c6dd00c4e01 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -110,7 +110,7 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { tryDelete(); - List threads = new ArrayList<>(); + List threads = new ArrayList<>(); int threadCount = 2; int batchSize = 1; if (random().nextBoolean()) { @@ -126,7 +126,7 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { } for (int i = 0; i < threadCount; i++) { - StopableIndexingThread indexThread = new StopableIndexingThread(controlClient, cloudClient, Integer.toString(i), true, maxUpdates, batchSize, pauseBetweenUpdates); // random().nextInt(999) + 1 + StoppableIndexingThread indexThread = new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true, maxUpdates, batchSize, pauseBetweenUpdates); // random().nextInt(999) + 1 threads.add(indexThread); indexThread.start(); } @@ -152,16 +152,16 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { chaosMonkey.stopTheMonkey(); } - for (StopableIndexingThread indexThread : threads) { + for (StoppableIndexingThread indexThread : threads) { indexThread.safeStop(); } // wait for stop... - for (StopableIndexingThread indexThread : threads) { + for (StoppableIndexingThread indexThread : threads) { indexThread.join(); } - for (StopableIndexingThread indexThread : threads) { + for (StoppableIndexingThread indexThread : threads) { assertEquals(0, indexThread.getFailCount()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index f828b933d37..b3961019d6c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -34,8 +34,8 @@ public class RecoveryZkTest extends AbstractFullDistribZkTestBase { //private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; private static Logger log = LoggerFactory.getLogger(RecoveryZkTest.class); - private StopableIndexingThread indexThread; - private StopableIndexingThread indexThread2; + private StoppableIndexingThread indexThread; + private StoppableIndexingThread indexThread2; public RecoveryZkTest() { super(); @@ -72,11 +72,11 @@ public class RecoveryZkTest extends AbstractFullDistribZkTestBase { maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)]; } - indexThread = new StopableIndexingThread(controlClient, cloudClient, "1", true, maxDoc, 1, true); + indexThread = new StoppableIndexingThread(controlClient, cloudClient, "1", true, maxDoc, 1, true); indexThread.start(); - indexThread2 = new StopableIndexingThread(controlClient, cloudClient, "2", true, maxDoc, 1, true); - + indexThread2 = new StoppableIndexingThread(controlClient, cloudClient, "2", true, maxDoc, 1, true); + indexThread2.start(); // give some time to index... diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java index 45946e4887e..e6bd148bda7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java @@ -29,7 +29,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.cloud.BasicDistributedZkTest; -import org.apache.solr.cloud.StopableIndexingThread; +import org.apache.solr.cloud.StoppableIndexingThread; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.HdfsDirectoryFactory; import org.apache.solr.core.SolrCore; @@ -100,18 +100,18 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest { waitForRecoveriesToFinish(ACOLLECTION + i, false); } List cloudClients = new ArrayList<>(); - List threads = new ArrayList<>(); + List threads = new ArrayList<>(); for (int i = 0; i < cnt; i++) { CloudSolrClient client = new CloudSolrClient(zkServer.getZkAddress()); client.setDefaultCollection(ACOLLECTION + i); cloudClients.add(client); - StopableIndexingThread indexThread = new StopableIndexingThread(null, client, "1", true, docCount, 1, true); + StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true); threads.add(indexThread); indexThread.start(); } int addCnt = 0; - for (StopableIndexingThread thread : threads) { + for (StoppableIndexingThread thread : threads) { thread.join(); addCnt += thread.getNumAdds() - thread.getNumDeletes(); } diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index 03e27481fce..21cb96fe9aa 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -230,6 +230,8 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { protected boolean verifyStress = true; protected int nThreads = 3; + protected int clientConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + protected int clientSoTimeout = 90000; public static int ORDERED = 1; public static int SKIP = 2; @@ -435,8 +437,8 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { try { // setup the client... HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/" + DEFAULT_TEST_CORENAME); - client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); - client.setSoTimeout(90000); + client.setConnectionTimeout(clientConnectionTimeout); + client.setSoTimeout(clientSoTimeout); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); return client; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java index 6fbde774c09..dd1394c0b82 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java @@ -232,4 +232,13 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes zkClient.printLayoutToStdOut(); zkClient.close(); } + + protected void restartZk(int pauseMillis) throws Exception { + log.info("Restarting ZK with a pause of {}ms in between", pauseMillis); + zkServer.shutdown(); + // disconnect enough to test stalling, if things stall, then clientSoTimeout w""ill be hit + Thread.sleep(pauseMillis); + zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); + zkServer.run(); + } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 17000fccf5d..b731d425cf5 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -77,7 +77,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; -import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -128,7 +127,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes protected boolean checkCreatedVsState; protected boolean useJettyDataDir = true; - protected Map proxies = new HashMap(); + protected Map proxies = new HashMap<>(); public static class CloudJettyRunner { public JettySolrRunner jetty; @@ -207,14 +206,11 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes if (isSSLMode()) { System.clearProperty("urlScheme"); - ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(), - AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT); - try { + try (ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(), + AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) { zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, - ZkStateReader.toJSON(Collections.singletonMap("urlScheme","https")), - CreateMode.PERSISTENT, true); - } finally { - zkStateReader.close(); + ZkStateReader.toJSON(Collections.singletonMap("urlScheme", "https")), + CreateMode.PERSISTENT, true); } } } @@ -1386,16 +1382,13 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes .getNumFound(); // do some really inefficient mapping... - ZkStateReader zk = new ZkStateReader(zkServer.getZkAddress(), 10000, - AbstractZkTestCase.TIMEOUT); Map slices = null; ClusterState clusterState; - try { + try (ZkStateReader zk = new ZkStateReader(zkServer.getZkAddress(), 10000, + AbstractZkTestCase.TIMEOUT)) { zk.createClusterStateWatchersAndUpdate(); clusterState = zk.getClusterState(); slices = clusterState.getSlicesMap(DEFAULT_COLLECTION); - } finally { - zk.close(); } if (slices == null) { @@ -1445,67 +1438,16 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION); - QueryResponse rsp = cloudClient.query(params); - return rsp; + return cloudClient.query(params); } - static abstract class StopableThread extends Thread { - public StopableThread(String name) { + static abstract class StoppableThread extends Thread { + public StoppableThread(String name) { super(name); } public abstract void safeStop(); } - class StopableSearchThread extends StopableThread { - private volatile boolean stop = false; - protected final AtomicInteger queryFails = new AtomicInteger(); - private String[] QUERIES = new String[] {"to come","their country","aid","co*"}; - - public StopableSearchThread() { - super("StopableSearchThread"); - setDaemon(true); - } - - @Override - public void run() { - Random random = random(); - int numSearches = 0; - - while (true && !stop) { - numSearches++; - try { - //to come to the aid of their country. - cloudClient.query(new SolrQuery(QUERIES[random.nextInt(QUERIES.length)])); - } catch (Exception e) { - System.err.println("QUERY REQUEST FAILED:"); - e.printStackTrace(); - if (e instanceof SolrServerException) { - System.err.println("ROOT CAUSE:"); - ((SolrServerException) e).getRootCause().printStackTrace(); - } - queryFails.incrementAndGet(); - } - try { - Thread.sleep(random.nextInt(4000) + 300); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - log.info("num searches done:" + numSearches + " with " + queryFails + " fails"); - } - - @Override - public void safeStop() { - stop = true; - } - - public int getFails() { - return queryFails.get(); - } - - }; - public void waitForThingsToLevelOut(int waitForRecTimeSeconds) throws Exception { log.info("Wait for recoveries to finish - wait " + waitForRecTimeSeconds + " for each attempt"); int cnt = 0; @@ -1934,14 +1876,14 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes REPLICATION_FACTOR, replicationFactor, MAX_SHARDS_PER_NODE, maxShardsPerNode, NUM_SLICES, numShards); - Map> collectionInfos = new HashMap>(); + Map> collectionInfos = new HashMap<>(); createCollection(collectionInfos, collName, props, client); } protected List ensureAllReplicasAreActive(String testCollectionName, String shardId, int shards, int rf, int maxWaitSecs) throws Exception { long startMs = System.currentTimeMillis(); - Map notLeaders = new HashMap(); + Map notLeaders = new HashMap<>(); ZkStateReader zkr = cloudClient.getZkStateReader(); zkr.updateClusterState(true); // force the state to be fresh @@ -2001,7 +1943,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes long diffMs = (System.currentTimeMillis() - startMs); log.info("Took " + diffMs + " ms to see all replicas become active."); - List replicas = new ArrayList(); + List replicas = new ArrayList<>(); replicas.addAll(notLeaders.values()); return replicas; } @@ -2017,7 +1959,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes if (collection != null) { cs = clusterState.getCollection(collection).toString(); } else { - Map map = new HashMap(); + Map map = new HashMap<>(); for (String coll : clusterState.getCollections()) map.put(coll, clusterState.getCollection(coll)); CharArr out = new CharArr(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java similarity index 93% rename from solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java rename to solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java index f6832112477..7a870c2dbf6 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/StopableIndexingThread.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java @@ -28,7 +28,7 @@ import org.apache.solr.common.SolrInputDocument; * limitations under the License. */ -public class StopableIndexingThread extends AbstractFullDistribZkTestBase.StopableThread { +public class StoppableIndexingThread extends AbstractFullDistribZkTestBase.StoppableThread { static String t1 = "a_t"; static String i1 = "a_i"; private volatile boolean stop = false; @@ -46,12 +46,12 @@ public class StopableIndexingThread extends AbstractFullDistribZkTestBase.Stopab private int batchSize; private boolean pauseBetweenUpdates; - public StopableIndexingThread(SolrClient controlClient, SolrClient cloudClient, String id, boolean doDeletes) { + public StoppableIndexingThread(SolrClient controlClient, SolrClient cloudClient, String id, boolean doDeletes) { this(controlClient, cloudClient, id, doDeletes, -1, 1, true); } - public StopableIndexingThread(SolrClient controlClient, SolrClient cloudClient, String id, boolean doDeletes, int numCycles, int batchSize, boolean pauseBetweenUpdates) { - super("StopableIndexingThread"); + public StoppableIndexingThread(SolrClient controlClient, SolrClient cloudClient, String id, boolean doDeletes, int numCycles, int batchSize, boolean pauseBetweenUpdates) { + super("StoppableIndexingThread"); this.controlClient = controlClient; this.cloudClient = cloudClient; this.id = id; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/StoppableSearchThread.java b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableSearchThread.java new file mode 100644 index 00000000000..d3eb0396af1 --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableSearchThread.java @@ -0,0 +1,82 @@ +package org.apache.solr.cloud; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class StoppableSearchThread extends AbstractFullDistribZkTestBase.StoppableThread { + static Logger log = LoggerFactory.getLogger(AbstractFullDistribZkTestBase.class); + + private final CloudSolrClient cloudClient; + private volatile boolean stop = false; + protected final AtomicInteger queryFails = new AtomicInteger(); + private String[] QUERIES = new String[] {"to come","their country","aid","co*"}; + + public StoppableSearchThread(CloudSolrClient cloudClient) { + super("StoppableSearchThread"); + this.cloudClient = cloudClient; + setDaemon(true); + } + + @Override + public void run() { + Random random = LuceneTestCase.random(); + int numSearches = 0; + + while (true && !stop) { + numSearches++; + try { + //to come to the aid of their country. + cloudClient.query(new SolrQuery(QUERIES[random.nextInt(QUERIES.length)])); + } catch (Exception e) { + System.err.println("QUERY REQUEST FAILED:"); + e.printStackTrace(); + if (e instanceof SolrServerException) { + System.err.println("ROOT CAUSE:"); + ((SolrServerException) e).getRootCause().printStackTrace(); + } + queryFails.incrementAndGet(); + } + try { + Thread.sleep(random.nextInt(4000) + 300); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + log.info("num searches done:" + numSearches + " with " + queryFails + " fails"); + } + + @Override + public void safeStop() { + stop = true; + } + + public int getFails() { + return queryFails.get(); + } + +}