simplify + improve test infra

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1364923 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2012-07-24 07:20:20 +00:00
parent 18587e58c9
commit 845a3c3d60
4 changed files with 83 additions and 77 deletions

View File

@ -71,10 +71,8 @@ public class ChaosMonkey {
public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader,
String collection, Map<String,List<CloudJettyRunner>> shardToJetty,
Map<String,List<SolrServer>> shardToClient,
Map<String,CloudJettyRunner> shardToLeaderJetty) {
this.shardToJetty = shardToJetty;
this.shardToClient = shardToClient;
this.shardToLeaderJetty = shardToLeaderJetty;
this.zkServer = zkServer;
this.zkStateReader = zkStateReader;

View File

@ -98,15 +98,12 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
protected volatile CloudSolrServer cloudClient;
protected Map<JettySolrRunner,ZkNodeProps> jettyToInfo = new HashMap<JettySolrRunner,ZkNodeProps>();
protected Map<CloudSolrServerClient,ZkNodeProps> clientToInfo = new HashMap<CloudSolrServerClient,ZkNodeProps>();
protected Map<String,List<SolrServer>> shardToClient = new HashMap<String,List<SolrServer>>();
protected List<CloudJettyRunner> cloudJettys = new ArrayList<CloudJettyRunner>();
protected Map<String,List<CloudJettyRunner>> shardToJetty = new HashMap<String,List<CloudJettyRunner>>();
private AtomicInteger jettyIntCntr = new AtomicInteger(0);
protected ChaosMonkey chaosMonkey;
protected volatile ZkStateReader zkStateReader;
protected Map<String,SolrServer> shardToLeaderClient = new HashMap<String,SolrServer>();
protected Map<String,CloudJettyRunner> shardToLeaderJetty = new HashMap<String,CloudJettyRunner>();
static class CloudJettyRunner {
@ -115,12 +112,14 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
String coreNodeName;
String url;
CloudSolrServerClient client;
public ZkNodeProps info;
}
static class CloudSolrServerClient {
SolrServer solrClient;
String shardName;
int port;
public ZkNodeProps info;
public CloudSolrServerClient() {}
@ -191,7 +190,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
}
chaosMonkey = new ChaosMonkey(zkServer, zkStateReader,
DEFAULT_COLLECTION, shardToJetty, shardToClient,
DEFAULT_COLLECTION, shardToJetty,
shardToLeaderJetty);
}
@ -337,9 +336,8 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
protected void updateMappingsFromZk(List<JettySolrRunner> jettys,
List<SolrServer> clients) throws Exception {
zkStateReader.updateCloudState(true);
shardToClient.clear();
cloudJettys.clear();
shardToJetty.clear();
jettyToInfo.clear();
CloudState cloudState = zkStateReader.getCloudState();
Map<String,Slice> slices = cloudState.getSlices(DEFAULT_COLLECTION);
@ -349,6 +347,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
+ DEFAULT_COLLECTION + " in " + cloudState.getCollections());
}
List<CloudSolrServerClient> theClients = new ArrayList<CloudSolrServerClient>();
for (SolrServer client : clients) {
// find info for this client in zk
nextClient:
@ -364,34 +363,20 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
csc.solrClient = client;
csc.port = port;
csc.shardName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
boolean isLeader = shard.getValue().containsKey(
ZkStateReader.LEADER_PROP);
clientToInfo.put(csc, shard.getValue());
List<SolrServer> list = shardToClient.get(slice.getKey());
if (list == null) {
list = new ArrayList<SolrServer>();
shardToClient.put(slice.getKey(), list);
}
list.add(client);
csc.info = shard.getValue();
theClients .add(csc);
if (isLeader) {
shardToLeaderClient.put(slice.getKey(), client);
}
break nextClient;
}
}
}
}
for (Map.Entry<String,Slice> slice : slices.entrySet()) {
// check that things look right
assertEquals(slice.getValue().getShards().size(), shardToClient.get(slice.getKey()).size());
}
for (JettySolrRunner jetty : jettys) {
int port = jetty.getLocalPort();
if (port == -1) {
continue; // If we cannot get the port, this jetty is down
throw new RuntimeException("Cannot find the port for jetty");
}
nextJetty:
@ -399,7 +384,6 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
if (shard.getKey().contains(":" + port + "_")) {
jettyToInfo.put(jetty, shard.getValue());
List<CloudJettyRunner> list = shardToJetty.get(slice.getKey());
if (list == null) {
list = new ArrayList<CloudJettyRunner>();
@ -409,14 +393,16 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
ZkStateReader.LEADER_PROP);
CloudJettyRunner cjr = new CloudJettyRunner();
cjr.jetty = jetty;
cjr.info = shard.getValue();
cjr.nodeName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
cjr.coreNodeName = shard.getKey();
cjr.url = shard.getValue().get(ZkStateReader.BASE_URL_PROP) + "/" + shard.getValue().get(ZkStateReader.CORE_NAME_PROP);
cjr.client = findClientByPort(port);
cjr.client = findClientByPort(port, theClients);
list.add(cjr);
if (isLeader) {
shardToLeaderJetty.put(slice.getKey(), cjr);
}
cloudJettys.add(cjr);
break nextJetty;
}
}
@ -435,8 +421,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
}
}
private CloudSolrServerClient findClientByPort(int port) {
Set<CloudSolrServerClient> theClients = clientToInfo.keySet();
private CloudSolrServerClient findClientByPort(int port, List<CloudSolrServerClient> theClients) {
for (CloudSolrServerClient client : theClients) {
if (client.port == port) {
return client;
@ -661,9 +646,9 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
// new server should be part of first shard
// how many docs are on the new shard?
for (SolrServer client : shardToClient.get("shard1")) {
for (CloudJettyRunner cjetty : shardToJetty.get("shard1")) {
if (VERBOSE) System.err.println("total:"
+ client.query(new SolrQuery("*:*")).getResults().getNumFound());
+ cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
}
checkShardConsistency("shard1");
@ -687,15 +672,21 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
commit();
long deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
long deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
.query(query).getResults().getNumFound();
query("q", "*:*", "sort", "n_tl1 desc");
// kill a shard
CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
cloudClient.connect();
int tries = 0;
while (cloudClient.getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
while (cloudClient
.getZkStateReader()
.getCloudState()
.liveNodesContain(
shardToJetty.get(SHARD2).get(0).info
.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 60) {
fail("Shard still reported as live in zk");
}
@ -724,7 +715,15 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
tries = 0;
while (((SolrDispatchFilter) shardToJetty.get(SHARD2).get(1).jetty.getDispatchFilter().getFilter()).getCores().getZkController().getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
while (((SolrDispatchFilter) shardToJetty.get(SHARD2).get(1).jetty
.getDispatchFilter().getFilter())
.getCores()
.getZkController()
.getZkStateReader()
.getCloudState()
.liveNodesContain(
shardToJetty.get(SHARD2).get(0).info
.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
@ -733,7 +732,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
index_specific(shardToClient.get(SHARD2).get(1), id, 1000, i1, 108, t1,
index_specific(shardToJetty.get(SHARD2).get(1).client.solrClient, id, 1000, i1, 108, t1,
"specific doc!");
commit();
@ -798,7 +797,8 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
waitForRecoveriesToFinish(false);
deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
.query(query).getResults().getNumFound();
// if we properly recovered, we should now have the couple missing docs that
// came in while shard was down
checkShardConsistency(true, false);
@ -1025,10 +1025,10 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
protected String checkShardConsistency(String shard, boolean verbose)
throws Exception {
List<SolrServer> solrClients = shardToClient.get(shard);
if (solrClients == null) {
List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
if (solrJetties == null) {
throw new RuntimeException("shard not found:" + shard + " keys:"
+ shardToClient.keySet());
+ shardToJetty.keySet());
}
long num = -1;
long lastNum = -1;
@ -1040,18 +1040,18 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
"The client count does not match up with the shard count for slice:"
+ shard,
zkStateReader.getCloudState().getSlice(DEFAULT_COLLECTION, shard)
.getShards().size(), solrClients.size());
.getShards().size(), solrJetties.size());
SolrServer lastClient = null;
for (SolrServer client : solrClients) {
ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(client));
for (CloudJettyRunner cjetty : solrJetties) {
ZkNodeProps props = cjetty.info;
if (verbose) System.err.println("client" + cnt++);
if (verbose) System.err.println("PROPS:" + props);
try {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
num = client.query(query).getResults().getNumFound();
num = cjetty.client.solrClient.query(query).getResults().getNumFound();
} catch (SolrServerException e) {
if (verbose) System.err.println("error contacting client: "
+ e.getMessage() + "\n");
@ -1076,7 +1076,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
if (active && live) {
if (lastNum > -1 && lastNum != num && failMessage == null) {
failMessage = shard + " is not consistent. Got " + lastNum + " from " + lastClient + "lastClient"
+ " and got " + num + " from " + client;
+ " and got " + num + " from " + cjetty.url;
if (verbose || true) {
System.err.println("######" + failMessage);
@ -1087,14 +1087,14 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
query.set("sort","id asc");
SolrDocumentList lst1 = lastClient.query(query).getResults();
SolrDocumentList lst2 = client.query(query).getResults();
SolrDocumentList lst2 = cjetty.client.solrClient.query(query).getResults();
showDiff(lst1, lst2, lastClient.toString(), client.toString());
showDiff(lst1, lst2, lastClient.toString(), cjetty.client.solrClient.toString());
}
}
lastNum = num;
lastClient = client;
lastClient = cjetty.client.solrClient;
}
}
return failMessage;
@ -1141,7 +1141,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
updateMappingsFromZk(jettys, clients);
Set<String> theShards = shardToClient.keySet();
Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
String shardFailMessage = checkShardConsistency(shard, verbose);
@ -1156,15 +1156,15 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
if (checkVsControl) {
// now check that the right # are on each shard
theShards = shardToClient.keySet();
theShards = shardToJetty.keySet();
int cnt = 0;
for (String s : theShards) {
int times = shardToClient.get(s).size();
int times = shardToJetty.get(s).size();
for (int i = 0; i < times; i++) {
try {
SolrServer client = shardToClient.get(s).get(i);
ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(
client));
CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
ZkNodeProps props = cjetty.info;
SolrServer client = cjetty.client.solrClient;
boolean active = props.get(ZkStateReader.STATE_PROP).equals(
ZkStateReader.ACTIVE);
if (active) {
@ -1196,7 +1196,8 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
}
private SolrServer getClient(String nodeName) {
for (CloudSolrServerClient client : clientToInfo.keySet()) {
for (CloudJettyRunner cjetty : cloudJettys) {
CloudSolrServerClient client = cjetty.client;
if (client.shardName.equals(nodeName)) {
return client.solrClient;
}
@ -1232,12 +1233,13 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
+ DEFAULT_COLLECTION + " in " + cloudState.getCollections());
}
for (SolrServer client : clients) {
for (CloudJettyRunner cjetty : cloudJettys) {
CloudSolrServerClient client = cjetty.client;
for (Map.Entry<String,Slice> slice : slices.entrySet()) {
Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
String shardName = new URI(
((HttpSolrServer) client).getBaseURL()).getPort()
((HttpSolrServer) client.solrClient).getBaseURL()).getPort()
+ "_solr_";
if (verbose && shard.getKey().endsWith(shardName)) {
System.err.println("shard:" + slice.getKey());
@ -1247,13 +1249,14 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
}
long count = 0;
String currentState = clientToInfo.get(new CloudSolrServerClient(client))
.get(ZkStateReader.STATE_PROP);
if (currentState != null && currentState.equals(ZkStateReader.ACTIVE) && zkStateReader.getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(client))
.get(ZkStateReader.NODE_NAME_PROP))) {
String currentState = cjetty.info.get(ZkStateReader.STATE_PROP);
if (currentState != null
&& currentState.equals(ZkStateReader.ACTIVE)
&& zkStateReader.getCloudState().liveNodesContain(
cjetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
count = client.query(query).getResults().getNumFound();
count = client.solrClient.query(query).getResults().getNumFound();
}
if (verbose) System.err.println("client docs:" + count + "\n\n");
@ -1365,7 +1368,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
updateMappingsFromZk(jettys, clients);
Set<String> theShards = shardToClient.keySet();
Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
failMessage = checkShardConsistency(shard, false);

View File

@ -100,8 +100,8 @@ public class RecoveryZkTest extends FullSolrCloudTest {
checkShardConsistency("shard1", false);
SolrQuery query = new SolrQuery("*:*");
query.setParam("distrib", "false");
long client1Docs = shardToClient.get("shard1").get(0).query(query).getResults().getNumFound();
long client2Docs = shardToClient.get("shard1").get(1).query(query).getResults().getNumFound();
long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();
long client2Docs = shardToJetty.get("shard1").get(1).client.solrClient.query(query).getResults().getNumFound();
assertTrue(client1Docs > 0);
assertEquals(client1Docs, client2Docs);

View File

@ -26,13 +26,10 @@ import java.util.Set;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.FullSolrCloudTest.CloudSolrServerClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@ -122,7 +119,8 @@ public class SyncSliceTest extends FullSolrCloudTest {
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
String baseUrl = ((HttpSolrServer) shardToClient.get("shard1").get(2)).getBaseURL();
String baseUrl = ((HttpSolrServer) shardToJetty.get("shard1").get(2).client.solrClient)
.getBaseURL();
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
@ -145,21 +143,28 @@ public class SyncSliceTest extends FullSolrCloudTest {
// kill the leader - new leader could have all the docs or be missing one
CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");
CloudSolrServerClient leaderClient = leaderJetty.client;
Set<JettySolrRunner> jetties = new HashSet<JettySolrRunner>();
Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
for (int i = 0; i < shardCount; i++) {
jetties.add(shardToJetty.get("shard1").get(i).jetty);
jetties.add(shardToJetty.get("shard1").get(i));
}
jetties.remove(leaderJetty);
chaosMonkey.killJetty(leaderJetty);
JettySolrRunner upJetty = jetties.iterator().next();
CloudJettyRunner upJetty = jetties.iterator().next();
// we are careful to make sure the downed node is no longer in the state,
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
assertNotNull(upJetty.jetty.getDispatchFilter());
assertNotNull(upJetty.jetty.getDispatchFilter());
assertNotNull(upJetty.jetty.getDispatchFilter().getFilter());
int tries = 0;
while (((SolrDispatchFilter) upJetty.getDispatchFilter().getFilter()).getCores().getZkController().getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(leaderClient).get(ZkStateReader.NODE_NAME_PROP))) {
while (((SolrDispatchFilter) upJetty.jetty.getDispatchFilter().getFilter())
.getCores().getZkController().getZkStateReader().getCloudState()
.liveNodesContain(leaderJetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
@ -184,7 +189,7 @@ public class SyncSliceTest extends FullSolrCloudTest {
updateMappingsFromZk(jettys, clients);
Set<String> theShards = shardToClient.keySet();
Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
failMessage = checkShardConsistency(shard, false);