mirror of https://github.com/apache/lucene.git
SOLR-10277: On 'downnode', lots of wasteful mutations are done to ZK
This commit is contained in:
Shalin Shekhar Mangar
parent
f08889f390
commit
60303028de
|
|
@ -207,6 +207,9 @@ Bug Fixes
|
|||
* SOLR-10416: The JSON output of /admin/metrics is fixed to write the container as a
|
||||
map (SimpleOrderedMap) instead of an array (NamedList). (shalin)
|
||||
|
||||
* SOLR-10277: On 'downnode', lots of wasteful mutations are done to ZK.
|
||||
(Joshua Humphries, Scott Blum, Varun Thacker, shalin)
|
||||
|
||||
================== 6.5.0 ==================
|
||||
|
||||
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
|
||||
|
|
|
|||
|
|
@ -383,7 +383,7 @@ public class Overseer implements Closeable {
|
|||
}
|
||||
break;
|
||||
case DOWNNODE:
|
||||
return new NodeMutator(getZkStateReader()).downNode(clusterState, message);
|
||||
return new NodeMutator().downNode(clusterState, message);
|
||||
default:
|
||||
throw new RuntimeException("unknown operation:" + operation + " contents:" + message.getProperties());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package org.apache.solr.cloud.overseer;
|
|||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -38,44 +37,44 @@ public class NodeMutator {
|
|||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
public NodeMutator(ZkStateReader zkStateReader) {
|
||||
|
||||
}
|
||||
|
||||
public List<ZkWriteCommand> downNode(ClusterState clusterState, ZkNodeProps message) {
|
||||
List<ZkWriteCommand> zkWriteCommands = new ArrayList<ZkWriteCommand>();
|
||||
List<ZkWriteCommand> zkWriteCommands = new ArrayList<>();
|
||||
String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);
|
||||
|
||||
log.debug("DownNode state invoked for node: " + nodeName);
|
||||
|
||||
Map<String, DocCollection> collections = clusterState.getCollectionsMap();
|
||||
for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
|
||||
String collection = entry.getKey();
|
||||
DocCollection docCollection = entry.getValue();
|
||||
|
||||
Map<String,Slice> slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap());
|
||||
|
||||
for (Entry<String,Slice> sliceEntry : slicesCopy.entrySet()) {
|
||||
Slice slice = docCollection.getSlice(sliceEntry.getKey());
|
||||
Map<String,Replica> newReplicas = new HashMap<String,Replica>();
|
||||
boolean needToUpdateCollection = false;
|
||||
for (Entry<String, Slice> sliceEntry : slicesCopy.entrySet()) {
|
||||
Slice slice = sliceEntry.getValue();
|
||||
Map<String, Replica> newReplicas = slice.getReplicasCopy();
|
||||
|
||||
Collection<Replica> replicas = slice.getReplicas();
|
||||
for (Replica replica : replicas) {
|
||||
Map<String,Object> props = replica.shallowCopy();
|
||||
String rNodeName = replica.getNodeName();
|
||||
if (rNodeName.equals(nodeName)) {
|
||||
log.debug("Update replica state for " + replica + " to " + Replica.State.DOWN.toString());
|
||||
Map<String, Object> props = replica.shallowCopy();
|
||||
props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
|
||||
Replica newReplica = new Replica(replica.getName(), props);
|
||||
newReplicas.put(replica.getName(), newReplica);
|
||||
needToUpdateCollection = true;
|
||||
}
|
||||
|
||||
Replica newReplica = new Replica(replica.getName(), props);
|
||||
newReplicas.put(replica.getName(), newReplica);
|
||||
}
|
||||
|
||||
Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy());
|
||||
slicesCopy.put(slice.getName(), newSlice);
|
||||
|
||||
}
|
||||
|
||||
zkWriteCommands.add(new ZkWriteCommand(entry.getKey(), docCollection.copyWithSlices(slicesCopy)));
|
||||
if (needToUpdateCollection) {
|
||||
zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy)));
|
||||
}
|
||||
}
|
||||
|
||||
return zkWriteCommands;
|
||||
|
|
|
|||
|
|
@ -41,5 +41,10 @@ public class ZkWriteCommand {
|
|||
public static ZkWriteCommand noop() {
|
||||
return new ZkWriteCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + ": " + (noop ? "no-op" : name + "=" + collection);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,233 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.cloud;
|
||||
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
|
||||
public class ClusterStateMockUtil {
|
||||
|
||||
private final static Pattern BLUEPRINT = Pattern.compile("([a-z])(\\d+)?(?:(['A','R','D','F']))?(\\*)?");
|
||||
|
||||
protected static class Result implements Closeable {
|
||||
OverseerAutoReplicaFailoverThread.DownReplica badReplica;
|
||||
ZkStateReader reader;
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected static ClusterStateMockUtil.Result buildClusterState(List<Result> results, String string, String ... liveNodes) {
|
||||
return buildClusterState(results, string, 1, liveNodes);
|
||||
}
|
||||
|
||||
protected static ClusterStateMockUtil.Result buildClusterState(List<Result> results, String string, int replicationFactor, String ... liveNodes) {
|
||||
return buildClusterState(results, string, replicationFactor, 10, liveNodes);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method lets you construct a complex ClusterState object by using simple strings of letters.
|
||||
*
|
||||
* c = collection, s = slice, r = replica, \d = node number (r2 means the replica is on node 2),
|
||||
* state = [A,R,D,F], * = replica to replace, binds to the left.
|
||||
*
|
||||
* For example:
|
||||
* csrr2rD*sr2csr
|
||||
*
|
||||
* Creates:
|
||||
*
|
||||
* 'csrr2rD*'
|
||||
* A collection, a shard, a replica on node 1 (the default) that is active (the default), a replica on node 2, and a replica on node 1
|
||||
* that has a state of down and is the replica we will be looking to put somewhere else (the *).
|
||||
*
|
||||
* 'sr2'
|
||||
* Then, another shard that has a replica on node 2.
|
||||
*
|
||||
* 'csr'
|
||||
* Then, another collection that has a shard with a single active replica on node 1.
|
||||
*
|
||||
* Result:
|
||||
* {
|
||||
* "collection2":{
|
||||
* "maxShardsPerNode":"1",
|
||||
* "replicationFactor":"1",
|
||||
* "shards":{"slice1":{
|
||||
* "state":"active",
|
||||
* "replicas":{"replica5":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"}}}}},
|
||||
* "collection1":{
|
||||
* "maxShardsPerNode":"1",
|
||||
* "replicationFactor":"1",
|
||||
* "shards":{
|
||||
* "slice1":{
|
||||
* "state":"active",
|
||||
* "replicas":{
|
||||
* "replica3 (bad)":{
|
||||
* "state":"down",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"},
|
||||
* "replica2":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl2_",
|
||||
* "base_url":"http://baseUrl2"},
|
||||
* "replica1":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"}}},
|
||||
* "slice2":{
|
||||
* "state":"active",
|
||||
* "replicas":{"replica4":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl2_",
|
||||
* "base_url":"http://baseUrl2"}}}}}}
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("resource")
|
||||
protected static ClusterStateMockUtil.Result buildClusterState(List<Result> results, String clusterDescription, int replicationFactor, int maxShardsPerNode, String ... liveNodes) {
|
||||
ClusterStateMockUtil.Result result = new ClusterStateMockUtil.Result();
|
||||
|
||||
Map<String,Slice> slices = null;
|
||||
Map<String,Replica> replicas = null;
|
||||
Map<String,Object> collectionProps = new HashMap<>();
|
||||
collectionProps.put(ZkStateReader.MAX_SHARDS_PER_NODE, Integer.toString(maxShardsPerNode));
|
||||
collectionProps.put(ZkStateReader.REPLICATION_FACTOR, Integer.toString(replicationFactor));
|
||||
Map<String,DocCollection> collectionStates = new HashMap<>();
|
||||
DocCollection docCollection = null;
|
||||
Slice slice = null;
|
||||
int replicaCount = 1;
|
||||
|
||||
Matcher m = BLUEPRINT.matcher(clusterDescription);
|
||||
while (m.find()) {
|
||||
Replica replica;
|
||||
switch (m.group(1)) {
|
||||
case "c":
|
||||
slices = new HashMap<>();
|
||||
docCollection = new DocCollection("collection" + (collectionStates.size() + 1), slices, collectionProps, null);
|
||||
collectionStates.put(docCollection.getName(), docCollection);
|
||||
break;
|
||||
case "s":
|
||||
replicas = new HashMap<>();
|
||||
slice = new Slice("slice" + (slices.size() + 1), replicas, null);
|
||||
slices.put(slice.getName(), slice);
|
||||
break;
|
||||
case "r":
|
||||
Map<String,Object> replicaPropMap = new HashMap<>();
|
||||
String node;
|
||||
|
||||
node = m.group(2);
|
||||
|
||||
if (node == null || node.trim().length() == 0) {
|
||||
node = "1";
|
||||
}
|
||||
|
||||
Replica.State state = Replica.State.ACTIVE;
|
||||
String stateCode = m.group(3);
|
||||
|
||||
if (stateCode != null) {
|
||||
switch (stateCode.charAt(0)) {
|
||||
case 'S':
|
||||
state = Replica.State.ACTIVE;
|
||||
break;
|
||||
case 'R':
|
||||
state = Replica.State.RECOVERING;
|
||||
break;
|
||||
case 'D':
|
||||
state = Replica.State.DOWN;
|
||||
break;
|
||||
case 'F':
|
||||
state = Replica.State.RECOVERY_FAILED;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
"Unexpected state for replica: " + stateCode);
|
||||
}
|
||||
}
|
||||
|
||||
String nodeName = "baseUrl" + node + "_";
|
||||
String replicaName = "replica" + replicaCount++;
|
||||
|
||||
if ("*".equals(m.group(4))) {
|
||||
replicaName += " (bad)";
|
||||
}
|
||||
|
||||
replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, nodeName);
|
||||
replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node);
|
||||
replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString());
|
||||
|
||||
replica = new Replica(replicaName, replicaPropMap);
|
||||
|
||||
if ("*".equals(m.group(4))) {
|
||||
result.badReplica = new OverseerAutoReplicaFailoverThread.DownReplica();
|
||||
result.badReplica.replica = replica;
|
||||
result.badReplica.slice = slice;
|
||||
result.badReplica.collection = docCollection;
|
||||
}
|
||||
|
||||
replicas.put(replica.getName(), replica);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ClusterState clusterState = new ClusterState(1, new HashSet<>(Arrays.asList(liveNodes)), collectionStates);
|
||||
MockZkStateReader reader = new MockZkStateReader(clusterState, collectionStates.keySet());
|
||||
|
||||
String json;
|
||||
try {
|
||||
json = new String(Utils.toJSON(clusterState), "UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException("Unexpected");
|
||||
}
|
||||
System.err.println(json);
|
||||
|
||||
// todo remove the limitation of always having a bad replica
|
||||
assert result.badReplica != null : "Is there no bad replica?";
|
||||
assert result.badReplica.slice != null : "Is there no bad replica?";
|
||||
|
||||
result.reader = reader;
|
||||
|
||||
if (results != null) {
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.cloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4Test;
|
||||
import org.apache.solr.cloud.overseer.NodeMutator;
|
||||
import org.apache.solr.cloud.overseer.ZkWriteCommand;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.ZkNodeProps;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.junit.Test;
|
||||
|
||||
public class NodeMutatorTest extends SolrTestCaseJ4Test {
|
||||
|
||||
private static final String NODE3 = "baseUrl3_";
|
||||
private static final String NODE3_URL = "http://baseUrl3";
|
||||
|
||||
private static final String NODE2 = "baseUrl2_";
|
||||
private static final String NODE2_URL = "http://baseUrl2";
|
||||
|
||||
private static final String NODE1 = "baseUrl1_";
|
||||
private static final String NODE1_URL = "http://baseUrl1";
|
||||
|
||||
@Test
|
||||
public void downNodeReportsAllImpactedCollectionsAndNothingElse() throws IOException {
|
||||
NodeMutator nm = new NodeMutator();
|
||||
ZkNodeProps props = new ZkNodeProps(ZkStateReader.NODE_NAME_PROP, NODE1);
|
||||
|
||||
//We use 2 nodes with maxShardsPerNode as 1
|
||||
//Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2
|
||||
//Collection2: 1 shard X 1 replica = replica1 on node2
|
||||
ClusterStateMockUtil.Result result = ClusterStateMockUtil.buildClusterState(null, "csrr2rD*csr2", 1, 1, NODE1, NODE2);
|
||||
ClusterState clusterState = result.reader.getClusterState();
|
||||
assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL);
|
||||
assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL);
|
||||
assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL);
|
||||
|
||||
props = new ZkNodeProps(ZkStateReader.NODE_NAME_PROP, NODE1);
|
||||
List<ZkWriteCommand> writes = nm.downNode(clusterState, props);
|
||||
assertEquals(writes.size(), 1);
|
||||
assertEquals(writes.get(0).name, "collection1");
|
||||
assertEquals(writes.get(0).collection.getReplica("replica1").getState(), Replica.State.DOWN);
|
||||
assertEquals(writes.get(0).collection.getReplica("replica2").getState(), Replica.State.ACTIVE);
|
||||
result.close();
|
||||
|
||||
//We use 3 nodes with maxShardsPerNode as 1
|
||||
//Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2
|
||||
//Collection2: 1 shard X 1 replica = replica1 on node2
|
||||
//Collection3: 1 shard X 3 replica = replica1 on node1 , replica2 on node2, replica3 on node3
|
||||
result = ClusterStateMockUtil.buildClusterState(null, "csrr2rD*csr2csr1r2r3", 1, 1, NODE1, NODE2, NODE3);
|
||||
clusterState = result.reader.getClusterState();
|
||||
assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL);
|
||||
assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL);
|
||||
|
||||
assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL);
|
||||
|
||||
assertEquals(clusterState.getCollection("collection3").getReplica("replica5").getBaseUrl(), NODE1_URL);
|
||||
assertEquals(clusterState.getCollection("collection3").getReplica("replica6").getBaseUrl(), NODE2_URL);
|
||||
assertEquals(clusterState.getCollection("collection3").getReplica("replica7").getBaseUrl(), NODE3_URL);
|
||||
|
||||
writes = nm.downNode(clusterState, props);
|
||||
assertEquals(writes.size(), 2);
|
||||
for (ZkWriteCommand write : writes) {
|
||||
if (write.name.equals("collection1")) {
|
||||
assertEquals(write.collection.getReplica("replica1").getState(), Replica.State.DOWN);
|
||||
assertEquals(write.collection.getReplica("replica2").getState(), Replica.State.ACTIVE);
|
||||
} else if (write.name.equals("collection3")) {
|
||||
assertEquals(write.collection.getReplica("replica5").getState(), Replica.State.DOWN);
|
||||
assertEquals(write.collection.getReplica("replica6").getState(), Replica.State.ACTIVE);
|
||||
assertEquals(write.collection.getReplica("replica7").getState(), Replica.State.ACTIVE);
|
||||
} else {
|
||||
fail("No other collection needs to be changed");
|
||||
}
|
||||
}
|
||||
result.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -16,30 +16,16 @@
|
|||
*/
|
||||
package org.apache.solr.cloud;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.DownReplica;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.solr.cloud.ClusterStateMockUtil.buildClusterState;
|
||||
|
||||
public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
|
||||
private static final String NODE6 = "baseUrl6_";
|
||||
private static final String NODE6_URL = "http://baseUrl6";
|
||||
|
|
@ -59,11 +45,7 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
|
|||
private static final String NODE1 = "baseUrl1_";
|
||||
private static final String NODE1_URL = "http://baseUrl1";
|
||||
|
||||
private final static Pattern BLUEPRINT = Pattern.compile("([a-z])(\\d+)?(?:(['A','R','D','F']))?(\\*)?");
|
||||
|
||||
private int buildNumber = 1;
|
||||
|
||||
private List<Result> results;
|
||||
private List<ClusterStateMockUtil.Result> results;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
|
|
@ -74,61 +56,50 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
|
|||
@After
|
||||
public void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
for (Result result : results) {
|
||||
for (ClusterStateMockUtil.Result result : results) {
|
||||
result.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBestCreateUrlBasics() {
|
||||
Result result = buildClusterState("csr1R*r2", NODE1);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr1R*r2", NODE1);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull("Should be no live node to failover to", createUrl);
|
||||
|
||||
result = buildClusterState("csr1R*r2", NODE1, NODE2);
|
||||
result = buildClusterState(results, "csr1R*r2", NODE1, NODE2);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull("Only failover candidate node already has a replica", createUrl);
|
||||
|
||||
result = buildClusterState("csr1R*r2sr3", NODE1, NODE2, NODE3);
|
||||
result = buildClusterState(results, "csr1R*r2sr3", NODE1, NODE2, NODE3);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals("Node3 does not have a replica from the bad slice and should be the best choice", NODE3_URL, createUrl);
|
||||
|
||||
result = buildClusterState("csr1R*r2Fsr3r4r5", NODE1, NODE2, NODE3);
|
||||
result = buildClusterState(results, "csr1R*r2Fsr3r4r5", NODE1, NODE2, NODE3);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertTrue(createUrl.equals(NODE3_URL));
|
||||
|
||||
result = buildClusterState("csr1*r2r3sr3r3sr4", NODE1, NODE2, NODE3, NODE4);
|
||||
result = buildClusterState(results, "csr1*r2r3sr3r3sr4", NODE1, NODE2, NODE3, NODE4);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE4_URL, createUrl);
|
||||
|
||||
result = buildClusterState("csr1*r2sr3r3sr4sr4", NODE1, NODE2, NODE3, NODE4);
|
||||
result = buildClusterState(results, "csr1*r2sr3r3sr4sr4", NODE1, NODE2, NODE3, NODE4);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL));
|
||||
}
|
||||
|
||||
|
||||
private static class Result implements Closeable {
|
||||
DownReplica badReplica;
|
||||
ZkStateReader reader;
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBestCreateUrlMultipleCollections() throws Exception {
|
||||
|
||||
Result result = buildClusterState("csr*r2csr2", NODE1);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2csr2", NODE1);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*r2csr2", NODE1);
|
||||
result = buildClusterState(results, "csr*r2csr2", NODE1);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*r2csr2", NODE1, NODE2);
|
||||
result = buildClusterState(results, "csr*r2csr2", NODE1, NODE2);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull(createUrl);
|
||||
}
|
||||
|
|
@ -136,11 +107,11 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
|
|||
@Test
|
||||
public void testGetBestCreateUrlMultipleCollections2() {
|
||||
|
||||
Result result = buildClusterState("csr*r2sr3cr2", NODE1);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2sr3cr2", NODE1);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*r2sr3cr2", NODE1, NODE2, NODE3);
|
||||
result = buildClusterState(results, "csr*r2sr3cr2", NODE1, NODE2, NODE3);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE3_URL, createUrl);
|
||||
}
|
||||
|
|
@ -148,253 +119,73 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
|
|||
|
||||
@Test
|
||||
public void testGetBestCreateUrlMultipleCollections3() {
|
||||
Result result = buildClusterState("csr5r1sr4r2sr3r6csr2*r6sr5r3sr4r3", NODE1, NODE4, NODE5, NODE6);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr5r1sr4r2sr3r6csr2*r6sr5r3sr4r3", NODE1, NODE4, NODE5, NODE6);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE1_URL, createUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetBestCreateUrlMultipleCollections4() {
|
||||
Result result = buildClusterState("csr1r4sr3r5sr2r6csr5r6sr4r6sr5*r4", NODE6);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr1r4sr3r5sr2r6csr5r6sr4r6sr5*r4", NODE6);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE6_URL, createUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailOverToEmptySolrInstance() {
|
||||
Result result = buildClusterState("csr1*r1sr1csr1", NODE2);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr1*r1sr1csr1", NODE2);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE2_URL, createUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFavorForeignSlices() {
|
||||
Result result = buildClusterState("csr*sr2csr3r3", NODE2, NODE3);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2csr3r3", NODE2, NODE3);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE3_URL, createUrl);
|
||||
|
||||
result = buildClusterState("csr*sr2csr3r3r3r3r3r3r3", NODE2, NODE3);
|
||||
result = buildClusterState(results, "csr*sr2csr3r3r3r3r3r3r3", NODE2, NODE3);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE2_URL, createUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCollectionMaxNodesPerShard() {
|
||||
Result result = buildClusterState("csr*sr2", 1, 1, NODE2);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*sr2", 1, 2, NODE2);
|
||||
result = buildClusterState(results, "csr*sr2", 1, 2, NODE2);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE2_URL, createUrl);
|
||||
|
||||
result = buildClusterState("csr*csr2r2", 1, 1, NODE2);
|
||||
result = buildClusterState(results, "csr*csr2r2", 1, 1, NODE2);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
|
||||
assertEquals(NODE2_URL, createUrl);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMaxCoresPerNode() {
|
||||
Result result = buildClusterState("csr*sr2", 1, 1, NODE2);
|
||||
ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2);
|
||||
String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1);
|
||||
assertNull(createUrl);
|
||||
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*sr2", 1, 2, NODE2);
|
||||
result = buildClusterState(results, "csr*sr2", 1, 2, NODE2);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
|
||||
assertEquals(NODE2_URL, createUrl);
|
||||
|
||||
result = buildClusterState("csr*sr2sr3sr4", 1, 1, NODE2, NODE3, NODE4);
|
||||
result = buildClusterState(results, "csr*sr2sr3sr4", 1, 1, NODE2, NODE3, NODE4);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1);
|
||||
assertNull(createUrl);
|
||||
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
|
||||
assertNull(createUrl);
|
||||
|
||||
result = buildClusterState("csr*sr2sr3sr4", 1, 2, NODE2, NODE3, NODE4);
|
||||
result = buildClusterState(results, "csr*sr2sr3sr4", 1, 2, NODE2, NODE3, NODE4);
|
||||
createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
|
||||
assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL));
|
||||
}
|
||||
|
||||
private Result buildClusterState(String string, String ... liveNodes) {
|
||||
return buildClusterState(string, 1, liveNodes);
|
||||
}
|
||||
|
||||
private Result buildClusterState(String string, int replicationFactor, String ... liveNodes) {
|
||||
return buildClusterState(string, replicationFactor, 10, liveNodes);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method lets you construct a complex ClusterState object by using simple strings of letters.
|
||||
*
|
||||
* c = collection, s = slice, r = replica, \d = node number (r2 means the replica is on node 2),
|
||||
* state = [A,R,D,F], * = replica to replace, binds to the left.
|
||||
*
|
||||
* For example:
|
||||
* csrr2rD*sr2csr
|
||||
*
|
||||
* Creates:
|
||||
*
|
||||
* 'csrr2rD*'
|
||||
* A collection, a shard, a replica on node 1 (the default) that is active (the default), a replica on node 2, and a replica on node 1
|
||||
* that has a state of down and is the replica we will be looking to put somewhere else (the *).
|
||||
*
|
||||
* 'sr2'
|
||||
* Then, another shard that has a replica on node 2.
|
||||
*
|
||||
* 'csr'
|
||||
* Then, another collection that has a shard with a single active replica on node 1.
|
||||
*
|
||||
* Result:
|
||||
* {
|
||||
* "collection2":{
|
||||
* "maxShardsPerNode":"1",
|
||||
* "replicationFactor":"1",
|
||||
* "shards":{"slice1":{
|
||||
* "state":"active",
|
||||
* "replicas":{"replica5":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"}}}}},
|
||||
* "collection1":{
|
||||
* "maxShardsPerNode":"1",
|
||||
* "replicationFactor":"1",
|
||||
* "shards":{
|
||||
* "slice1":{
|
||||
* "state":"active",
|
||||
* "replicas":{
|
||||
* "replica3 (bad)":{
|
||||
* "state":"down",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"},
|
||||
* "replica2":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl2_",
|
||||
* "base_url":"http://baseUrl2"},
|
||||
* "replica1":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl1_",
|
||||
* "base_url":"http://baseUrl1"}}},
|
||||
* "slice2":{
|
||||
* "state":"active",
|
||||
* "replicas":{"replica4":{
|
||||
* "state":"active",
|
||||
* "node_name":"baseUrl2_",
|
||||
* "base_url":"http://baseUrl2"}}}}}}
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("resource")
|
||||
private Result buildClusterState(String clusterDescription, int replicationFactor, int maxShardsPerNode, String ... liveNodes) {
|
||||
Result result = new Result();
|
||||
|
||||
Map<String,Slice> slices = null;
|
||||
Map<String,Replica> replicas = null;
|
||||
Map<String,Object> collectionProps = new HashMap<>();
|
||||
collectionProps.put(ZkStateReader.MAX_SHARDS_PER_NODE, Integer.toString(maxShardsPerNode));
|
||||
collectionProps.put(ZkStateReader.REPLICATION_FACTOR, Integer.toString(replicationFactor));
|
||||
Map<String,DocCollection> collectionStates = new HashMap<>();
|
||||
DocCollection docCollection = null;
|
||||
Slice slice = null;
|
||||
int replicaCount = 1;
|
||||
|
||||
Matcher m = BLUEPRINT.matcher(clusterDescription);
|
||||
while (m.find()) {
|
||||
Replica replica;
|
||||
switch (m.group(1)) {
|
||||
case "c":
|
||||
slices = new HashMap<>();
|
||||
docCollection = new DocCollection("collection" + (collectionStates.size() + 1), slices, collectionProps, null);
|
||||
collectionStates.put(docCollection.getName(), docCollection);
|
||||
break;
|
||||
case "s":
|
||||
replicas = new HashMap<>();
|
||||
slice = new Slice("slice" + (slices.size() + 1), replicas, null);
|
||||
slices.put(slice.getName(), slice);
|
||||
break;
|
||||
case "r":
|
||||
Map<String,Object> replicaPropMap = new HashMap<>();
|
||||
String node;
|
||||
|
||||
node = m.group(2);
|
||||
|
||||
if (node == null || node.trim().length() == 0) {
|
||||
node = "1";
|
||||
}
|
||||
|
||||
Replica.State state = Replica.State.ACTIVE;
|
||||
String stateCode = m.group(3);
|
||||
|
||||
if (stateCode != null) {
|
||||
switch (stateCode.charAt(0)) {
|
||||
case 'S':
|
||||
state = Replica.State.ACTIVE;
|
||||
break;
|
||||
case 'R':
|
||||
state = Replica.State.RECOVERING;
|
||||
break;
|
||||
case 'D':
|
||||
state = Replica.State.DOWN;
|
||||
break;
|
||||
case 'F':
|
||||
state = Replica.State.RECOVERY_FAILED;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
"Unexpected state for replica: " + stateCode);
|
||||
}
|
||||
}
|
||||
|
||||
String nodeName = "baseUrl" + node + "_";
|
||||
String replicaName = "replica" + replicaCount++;
|
||||
|
||||
if ("*".equals(m.group(4))) {
|
||||
replicaName += " (bad)";
|
||||
}
|
||||
|
||||
replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, nodeName);
|
||||
replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node);
|
||||
replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString());
|
||||
|
||||
replica = new Replica(replicaName, replicaPropMap);
|
||||
|
||||
if ("*".equals(m.group(4))) {
|
||||
result.badReplica = new DownReplica();
|
||||
result.badReplica.replica = replica;
|
||||
result.badReplica.slice = slice;
|
||||
result.badReplica.collection = docCollection;
|
||||
}
|
||||
|
||||
replicas.put(replica.getName(), replica);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// trunk briefly had clusterstate taking a zkreader :( this was required to work around that - leaving
|
||||
// until that issue is resolved.
|
||||
MockZkStateReader reader = new MockZkStateReader(null, collectionStates.keySet());
|
||||
ClusterState clusterState = new ClusterState(1, new HashSet<>(Arrays.asList(liveNodes)), collectionStates);
|
||||
reader = new MockZkStateReader(clusterState, collectionStates.keySet());
|
||||
|
||||
String json;
|
||||
try {
|
||||
json = new String(Utils.toJSON(clusterState), "UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException("Unexpected");
|
||||
}
|
||||
System.err.println("build:" + buildNumber++);
|
||||
System.err.println(json);
|
||||
|
||||
assert result.badReplica != null : "Is there no bad replica?";
|
||||
assert result.badReplica.slice != null : "Is there no bad replica?";
|
||||
|
||||
result.reader = reader;
|
||||
|
||||
results.add(result);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue