From 60303028debf3927e0c3abfaaa4015f73b88e689 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Wed, 5 Apr 2017 16:01:44 +0530 Subject: [PATCH] SOLR-10277: On 'downnode', lots of wasteful mutations are done to ZK --- solr/CHANGES.txt | 3 + .../java/org/apache/solr/cloud/Overseer.java | 2 +- .../solr/cloud/overseer/NodeMutator.java | 29 +- .../solr/cloud/overseer/ZkWriteCommand.java | 5 + .../solr/cloud/ClusterStateMockUtil.java | 233 ++++++++++++++++ .../apache/solr/cloud/NodeMutatorTest.java | 95 +++++++ .../SharedFSAutoReplicaFailoverUtilsTest.java | 263 ++---------------- 7 files changed, 378 insertions(+), 252 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ClusterStateMockUtil.java create mode 100644 solr/core/src/test/org/apache/solr/cloud/NodeMutatorTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4fa03534a21..35403159c53 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -207,6 +207,9 @@ Bug Fixes * SOLR-10416: The JSON output of /admin/metrics is fixed to write the container as a map (SimpleOrderedMap) instead of an array (NamedList). (shalin) +* SOLR-10277: On 'downnode', lots of wasteful mutations are done to ZK. + (Joshua Humphries, Scott Blum, Varun Thacker, shalin) + ================== 6.5.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 4d3cee7d737..f97fbaca833 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -383,7 +383,7 @@ public class Overseer implements Closeable { } break; case DOWNNODE: - return new NodeMutator(getZkStateReader()).downNode(clusterState, message); + return new NodeMutator().downNode(clusterState, message); default: throw new RuntimeException("unknown operation:" + operation + " contents:" + message.getProperties()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java index 0036fe1049c..55fd3efb618 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java @@ -19,7 +19,6 @@ package org.apache.solr.cloud.overseer; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,44 +37,44 @@ public class NodeMutator { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public NodeMutator(ZkStateReader zkStateReader) { - - } - public List downNode(ClusterState clusterState, ZkNodeProps message) { - List zkWriteCommands = new ArrayList(); + List zkWriteCommands = new ArrayList<>(); String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP); log.debug("DownNode state invoked for node: " + nodeName); Map collections = clusterState.getCollectionsMap(); for (Map.Entry entry : collections.entrySet()) { + String collection = entry.getKey(); DocCollection docCollection = entry.getValue(); + Map slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap()); - for (Entry sliceEntry : slicesCopy.entrySet()) { - Slice slice = docCollection.getSlice(sliceEntry.getKey()); - Map newReplicas = new HashMap(); + boolean needToUpdateCollection = false; + for (Entry sliceEntry : slicesCopy.entrySet()) { + Slice slice = sliceEntry.getValue(); + Map newReplicas = slice.getReplicasCopy(); Collection replicas = slice.getReplicas(); for (Replica replica : replicas) { - Map props = replica.shallowCopy(); String rNodeName = replica.getNodeName(); if (rNodeName.equals(nodeName)) { log.debug("Update replica state for " + replica + " to " + Replica.State.DOWN.toString()); + Map props = replica.shallowCopy(); props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString()); + Replica newReplica = new Replica(replica.getName(), props); + newReplicas.put(replica.getName(), newReplica); + needToUpdateCollection = true; } - - Replica newReplica = new Replica(replica.getName(), props); - newReplicas.put(replica.getName(), newReplica); } Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy()); slicesCopy.put(slice.getName(), newSlice); - } - zkWriteCommands.add(new ZkWriteCommand(entry.getKey(), docCollection.copyWithSlices(slicesCopy))); + if (needToUpdateCollection) { + zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy))); + } } return zkWriteCommands; diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java index 1697522e3c8..d464863df4b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java @@ -41,5 +41,10 @@ public class ZkWriteCommand { public static ZkWriteCommand noop() { return new ZkWriteCommand(); } + + @Override + public String toString() { + return getClass().getSimpleName() + ": " + (noop ? "no-op" : name + "=" + collection); + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ClusterStateMockUtil.java b/solr/core/src/test/org/apache/solr/cloud/ClusterStateMockUtil.java new file mode 100644 index 00000000000..e0cf3f733b0 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ClusterStateMockUtil.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + + +import java.io.Closeable; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.Utils; + +public class ClusterStateMockUtil { + + private final static Pattern BLUEPRINT = Pattern.compile("([a-z])(\\d+)?(?:(['A','R','D','F']))?(\\*)?"); + + protected static class Result implements Closeable { + OverseerAutoReplicaFailoverThread.DownReplica badReplica; + ZkStateReader reader; + + @Override + public void close() throws IOException { + reader.close(); + } + } + + protected static ClusterStateMockUtil.Result buildClusterState(List results, String string, String ... liveNodes) { + return buildClusterState(results, string, 1, liveNodes); + } + + protected static ClusterStateMockUtil.Result buildClusterState(List results, String string, int replicationFactor, String ... liveNodes) { + return buildClusterState(results, string, replicationFactor, 10, liveNodes); + } + + /** + * This method lets you construct a complex ClusterState object by using simple strings of letters. + * + * c = collection, s = slice, r = replica, \d = node number (r2 means the replica is on node 2), + * state = [A,R,D,F], * = replica to replace, binds to the left. + * + * For example: + * csrr2rD*sr2csr + * + * Creates: + * + * 'csrr2rD*' + * A collection, a shard, a replica on node 1 (the default) that is active (the default), a replica on node 2, and a replica on node 1 + * that has a state of down and is the replica we will be looking to put somewhere else (the *). + * + * 'sr2' + * Then, another shard that has a replica on node 2. + * + * 'csr' + * Then, another collection that has a shard with a single active replica on node 1. + * + * Result: + * { + * "collection2":{ + * "maxShardsPerNode":"1", + * "replicationFactor":"1", + * "shards":{"slice1":{ + * "state":"active", + * "replicas":{"replica5":{ + * "state":"active", + * "node_name":"baseUrl1_", + * "base_url":"http://baseUrl1"}}}}}, + * "collection1":{ + * "maxShardsPerNode":"1", + * "replicationFactor":"1", + * "shards":{ + * "slice1":{ + * "state":"active", + * "replicas":{ + * "replica3 (bad)":{ + * "state":"down", + * "node_name":"baseUrl1_", + * "base_url":"http://baseUrl1"}, + * "replica2":{ + * "state":"active", + * "node_name":"baseUrl2_", + * "base_url":"http://baseUrl2"}, + * "replica1":{ + * "state":"active", + * "node_name":"baseUrl1_", + * "base_url":"http://baseUrl1"}}}, + * "slice2":{ + * "state":"active", + * "replicas":{"replica4":{ + * "state":"active", + * "node_name":"baseUrl2_", + * "base_url":"http://baseUrl2"}}}}}} + * + */ + @SuppressWarnings("resource") + protected static ClusterStateMockUtil.Result buildClusterState(List results, String clusterDescription, int replicationFactor, int maxShardsPerNode, String ... liveNodes) { + ClusterStateMockUtil.Result result = new ClusterStateMockUtil.Result(); + + Map slices = null; + Map replicas = null; + Map collectionProps = new HashMap<>(); + collectionProps.put(ZkStateReader.MAX_SHARDS_PER_NODE, Integer.toString(maxShardsPerNode)); + collectionProps.put(ZkStateReader.REPLICATION_FACTOR, Integer.toString(replicationFactor)); + Map collectionStates = new HashMap<>(); + DocCollection docCollection = null; + Slice slice = null; + int replicaCount = 1; + + Matcher m = BLUEPRINT.matcher(clusterDescription); + while (m.find()) { + Replica replica; + switch (m.group(1)) { + case "c": + slices = new HashMap<>(); + docCollection = new DocCollection("collection" + (collectionStates.size() + 1), slices, collectionProps, null); + collectionStates.put(docCollection.getName(), docCollection); + break; + case "s": + replicas = new HashMap<>(); + slice = new Slice("slice" + (slices.size() + 1), replicas, null); + slices.put(slice.getName(), slice); + break; + case "r": + Map replicaPropMap = new HashMap<>(); + String node; + + node = m.group(2); + + if (node == null || node.trim().length() == 0) { + node = "1"; + } + + Replica.State state = Replica.State.ACTIVE; + String stateCode = m.group(3); + + if (stateCode != null) { + switch (stateCode.charAt(0)) { + case 'S': + state = Replica.State.ACTIVE; + break; + case 'R': + state = Replica.State.RECOVERING; + break; + case 'D': + state = Replica.State.DOWN; + break; + case 'F': + state = Replica.State.RECOVERY_FAILED; + break; + default: + throw new IllegalArgumentException( + "Unexpected state for replica: " + stateCode); + } + } + + String nodeName = "baseUrl" + node + "_"; + String replicaName = "replica" + replicaCount++; + + if ("*".equals(m.group(4))) { + replicaName += " (bad)"; + } + + replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, nodeName); + replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node); + replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString()); + + replica = new Replica(replicaName, replicaPropMap); + + if ("*".equals(m.group(4))) { + result.badReplica = new OverseerAutoReplicaFailoverThread.DownReplica(); + result.badReplica.replica = replica; + result.badReplica.slice = slice; + result.badReplica.collection = docCollection; + } + + replicas.put(replica.getName(), replica); + break; + default: + break; + } + } + + ClusterState clusterState = new ClusterState(1, new HashSet<>(Arrays.asList(liveNodes)), collectionStates); + MockZkStateReader reader = new MockZkStateReader(clusterState, collectionStates.keySet()); + + String json; + try { + json = new String(Utils.toJSON(clusterState), "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unexpected"); + } + System.err.println(json); + + // todo remove the limitation of always having a bad replica + assert result.badReplica != null : "Is there no bad replica?"; + assert result.badReplica.slice != null : "Is there no bad replica?"; + + result.reader = reader; + + if (results != null) { + results.add(result); + } + + return result; + } + + +} diff --git a/solr/core/src/test/org/apache/solr/cloud/NodeMutatorTest.java b/solr/core/src/test/org/apache/solr/cloud/NodeMutatorTest.java new file mode 100644 index 00000000000..ffa6ba2f567 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/NodeMutatorTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.io.IOException; +import java.util.List; + +import org.apache.solr.SolrTestCaseJ4Test; +import org.apache.solr.cloud.overseer.NodeMutator; +import org.apache.solr.cloud.overseer.ZkWriteCommand; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.Test; + +public class NodeMutatorTest extends SolrTestCaseJ4Test { + + private static final String NODE3 = "baseUrl3_"; + private static final String NODE3_URL = "http://baseUrl3"; + + private static final String NODE2 = "baseUrl2_"; + private static final String NODE2_URL = "http://baseUrl2"; + + private static final String NODE1 = "baseUrl1_"; + private static final String NODE1_URL = "http://baseUrl1"; + + @Test + public void downNodeReportsAllImpactedCollectionsAndNothingElse() throws IOException { + NodeMutator nm = new NodeMutator(); + ZkNodeProps props = new ZkNodeProps(ZkStateReader.NODE_NAME_PROP, NODE1); + + //We use 2 nodes with maxShardsPerNode as 1 + //Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2 + //Collection2: 1 shard X 1 replica = replica1 on node2 + ClusterStateMockUtil.Result result = ClusterStateMockUtil.buildClusterState(null, "csrr2rD*csr2", 1, 1, NODE1, NODE2); + ClusterState clusterState = result.reader.getClusterState(); + assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL); + assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL); + assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL); + + props = new ZkNodeProps(ZkStateReader.NODE_NAME_PROP, NODE1); + List writes = nm.downNode(clusterState, props); + assertEquals(writes.size(), 1); + assertEquals(writes.get(0).name, "collection1"); + assertEquals(writes.get(0).collection.getReplica("replica1").getState(), Replica.State.DOWN); + assertEquals(writes.get(0).collection.getReplica("replica2").getState(), Replica.State.ACTIVE); + result.close(); + + //We use 3 nodes with maxShardsPerNode as 1 + //Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2 + //Collection2: 1 shard X 1 replica = replica1 on node2 + //Collection3: 1 shard X 3 replica = replica1 on node1 , replica2 on node2, replica3 on node3 + result = ClusterStateMockUtil.buildClusterState(null, "csrr2rD*csr2csr1r2r3", 1, 1, NODE1, NODE2, NODE3); + clusterState = result.reader.getClusterState(); + assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL); + assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL); + + assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL); + + assertEquals(clusterState.getCollection("collection3").getReplica("replica5").getBaseUrl(), NODE1_URL); + assertEquals(clusterState.getCollection("collection3").getReplica("replica6").getBaseUrl(), NODE2_URL); + assertEquals(clusterState.getCollection("collection3").getReplica("replica7").getBaseUrl(), NODE3_URL); + + writes = nm.downNode(clusterState, props); + assertEquals(writes.size(), 2); + for (ZkWriteCommand write : writes) { + if (write.name.equals("collection1")) { + assertEquals(write.collection.getReplica("replica1").getState(), Replica.State.DOWN); + assertEquals(write.collection.getReplica("replica2").getState(), Replica.State.ACTIVE); + } else if (write.name.equals("collection3")) { + assertEquals(write.collection.getReplica("replica5").getState(), Replica.State.DOWN); + assertEquals(write.collection.getReplica("replica6").getState(), Replica.State.ACTIVE); + assertEquals(write.collection.getReplica("replica7").getState(), Replica.State.ACTIVE); + } else { + fail("No other collection needs to be changed"); + } + } + result.close(); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java index f5fee2174e3..342342020ec 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java @@ -16,30 +16,16 @@ */ package org.apache.solr.cloud; -import java.io.Closeable; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.DownReplica; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import static org.apache.solr.cloud.ClusterStateMockUtil.buildClusterState; + public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 { private static final String NODE6 = "baseUrl6_"; private static final String NODE6_URL = "http://baseUrl6"; @@ -58,12 +44,8 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 { private static final String NODE1 = "baseUrl1_"; private static final String NODE1_URL = "http://baseUrl1"; - - private final static Pattern BLUEPRINT = Pattern.compile("([a-z])(\\d+)?(?:(['A','R','D','F']))?(\\*)?"); - - private int buildNumber = 1; - private List results; + private List results; @Before public void setUp() throws Exception { @@ -74,61 +56,50 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 { @After public void tearDown() throws Exception { super.tearDown(); - for (Result result : results) { + for (ClusterStateMockUtil.Result result : results) { result.close(); } } @Test public void testGetBestCreateUrlBasics() { - Result result = buildClusterState("csr1R*r2", NODE1); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr1R*r2", NODE1); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull("Should be no live node to failover to", createUrl); - result = buildClusterState("csr1R*r2", NODE1, NODE2); + result = buildClusterState(results, "csr1R*r2", NODE1, NODE2); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull("Only failover candidate node already has a replica", createUrl); - result = buildClusterState("csr1R*r2sr3", NODE1, NODE2, NODE3); + result = buildClusterState(results, "csr1R*r2sr3", NODE1, NODE2, NODE3); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals("Node3 does not have a replica from the bad slice and should be the best choice", NODE3_URL, createUrl); - result = buildClusterState("csr1R*r2Fsr3r4r5", NODE1, NODE2, NODE3); + result = buildClusterState(results, "csr1R*r2Fsr3r4r5", NODE1, NODE2, NODE3); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertTrue(createUrl.equals(NODE3_URL)); - result = buildClusterState("csr1*r2r3sr3r3sr4", NODE1, NODE2, NODE3, NODE4); + result = buildClusterState(results, "csr1*r2r3sr3r3sr4", NODE1, NODE2, NODE3, NODE4); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE4_URL, createUrl); - result = buildClusterState("csr1*r2sr3r3sr4sr4", NODE1, NODE2, NODE3, NODE4); + result = buildClusterState(results, "csr1*r2sr3r3sr4sr4", NODE1, NODE2, NODE3, NODE4); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL)); } - - - private static class Result implements Closeable { - DownReplica badReplica; - ZkStateReader reader; - - @Override - public void close() throws IOException { - reader.close(); - } - } @Test public void testGetBestCreateUrlMultipleCollections() throws Exception { - Result result = buildClusterState("csr*r2csr2", NODE1); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2csr2", NODE1); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull(createUrl); - result = buildClusterState("csr*r2csr2", NODE1); + result = buildClusterState(results, "csr*r2csr2", NODE1); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull(createUrl); - result = buildClusterState("csr*r2csr2", NODE1, NODE2); + result = buildClusterState(results, "csr*r2csr2", NODE1, NODE2); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull(createUrl); } @@ -136,11 +107,11 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 { @Test public void testGetBestCreateUrlMultipleCollections2() { - Result result = buildClusterState("csr*r2sr3cr2", NODE1); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2sr3cr2", NODE1); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull(createUrl); - result = buildClusterState("csr*r2sr3cr2", NODE1, NODE2, NODE3); + result = buildClusterState(results, "csr*r2sr3cr2", NODE1, NODE2, NODE3); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE3_URL, createUrl); } @@ -148,253 +119,73 @@ public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 { @Test public void testGetBestCreateUrlMultipleCollections3() { - Result result = buildClusterState("csr5r1sr4r2sr3r6csr2*r6sr5r3sr4r3", NODE1, NODE4, NODE5, NODE6); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr5r1sr4r2sr3r6csr2*r6sr5r3sr4r3", NODE1, NODE4, NODE5, NODE6); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE1_URL, createUrl); } @Test public void testGetBestCreateUrlMultipleCollections4() { - Result result = buildClusterState("csr1r4sr3r5sr2r6csr5r6sr4r6sr5*r4", NODE6); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr1r4sr3r5sr2r6csr5r6sr4r6sr5*r4", NODE6); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE6_URL, createUrl); } @Test public void testFailOverToEmptySolrInstance() { - Result result = buildClusterState("csr1*r1sr1csr1", NODE2); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr1*r1sr1csr1", NODE2); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE2_URL, createUrl); } @Test public void testFavorForeignSlices() { - Result result = buildClusterState("csr*sr2csr3r3", NODE2, NODE3); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2csr3r3", NODE2, NODE3); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE3_URL, createUrl); - result = buildClusterState("csr*sr2csr3r3r3r3r3r3r3", NODE2, NODE3); + result = buildClusterState(results, "csr*sr2csr3r3r3r3r3r3r3", NODE2, NODE3); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE2_URL, createUrl); } @Test public void testCollectionMaxNodesPerShard() { - Result result = buildClusterState("csr*sr2", 1, 1, NODE2); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertNull(createUrl); - result = buildClusterState("csr*sr2", 1, 2, NODE2); + result = buildClusterState(results, "csr*sr2", 1, 2, NODE2); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE2_URL, createUrl); - result = buildClusterState("csr*csr2r2", 1, 1, NODE2); + result = buildClusterState(results, "csr*csr2r2", 1, 1, NODE2); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null); assertEquals(NODE2_URL, createUrl); } @Test public void testMaxCoresPerNode() { - Result result = buildClusterState("csr*sr2", 1, 1, NODE2); + ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2); String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1); assertNull(createUrl); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2); assertNull(createUrl); - result = buildClusterState("csr*sr2", 1, 2, NODE2); + result = buildClusterState(results, "csr*sr2", 1, 2, NODE2); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2); assertEquals(NODE2_URL, createUrl); - result = buildClusterState("csr*sr2sr3sr4", 1, 1, NODE2, NODE3, NODE4); + result = buildClusterState(results, "csr*sr2sr3sr4", 1, 1, NODE2, NODE3, NODE4); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1); assertNull(createUrl); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2); assertNull(createUrl); - result = buildClusterState("csr*sr2sr3sr4", 1, 2, NODE2, NODE3, NODE4); + result = buildClusterState(results, "csr*sr2sr3sr4", 1, 2, NODE2, NODE3, NODE4); createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2); assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL)); } - - private Result buildClusterState(String string, String ... liveNodes) { - return buildClusterState(string, 1, liveNodes); - } - - private Result buildClusterState(String string, int replicationFactor, String ... liveNodes) { - return buildClusterState(string, replicationFactor, 10, liveNodes); - } - - /** - * This method lets you construct a complex ClusterState object by using simple strings of letters. - * - * c = collection, s = slice, r = replica, \d = node number (r2 means the replica is on node 2), - * state = [A,R,D,F], * = replica to replace, binds to the left. - * - * For example: - * csrr2rD*sr2csr - * - * Creates: - * - * 'csrr2rD*' - * A collection, a shard, a replica on node 1 (the default) that is active (the default), a replica on node 2, and a replica on node 1 - * that has a state of down and is the replica we will be looking to put somewhere else (the *). - * - * 'sr2' - * Then, another shard that has a replica on node 2. - * - * 'csr' - * Then, another collection that has a shard with a single active replica on node 1. - * - * Result: - * { - * "collection2":{ - * "maxShardsPerNode":"1", - * "replicationFactor":"1", - * "shards":{"slice1":{ - * "state":"active", - * "replicas":{"replica5":{ - * "state":"active", - * "node_name":"baseUrl1_", - * "base_url":"http://baseUrl1"}}}}}, - * "collection1":{ - * "maxShardsPerNode":"1", - * "replicationFactor":"1", - * "shards":{ - * "slice1":{ - * "state":"active", - * "replicas":{ - * "replica3 (bad)":{ - * "state":"down", - * "node_name":"baseUrl1_", - * "base_url":"http://baseUrl1"}, - * "replica2":{ - * "state":"active", - * "node_name":"baseUrl2_", - * "base_url":"http://baseUrl2"}, - * "replica1":{ - * "state":"active", - * "node_name":"baseUrl1_", - * "base_url":"http://baseUrl1"}}}, - * "slice2":{ - * "state":"active", - * "replicas":{"replica4":{ - * "state":"active", - * "node_name":"baseUrl2_", - * "base_url":"http://baseUrl2"}}}}}} - * - */ - @SuppressWarnings("resource") - private Result buildClusterState(String clusterDescription, int replicationFactor, int maxShardsPerNode, String ... liveNodes) { - Result result = new Result(); - - Map slices = null; - Map replicas = null; - Map collectionProps = new HashMap<>(); - collectionProps.put(ZkStateReader.MAX_SHARDS_PER_NODE, Integer.toString(maxShardsPerNode)); - collectionProps.put(ZkStateReader.REPLICATION_FACTOR, Integer.toString(replicationFactor)); - Map collectionStates = new HashMap<>(); - DocCollection docCollection = null; - Slice slice = null; - int replicaCount = 1; - - Matcher m = BLUEPRINT.matcher(clusterDescription); - while (m.find()) { - Replica replica; - switch (m.group(1)) { - case "c": - slices = new HashMap<>(); - docCollection = new DocCollection("collection" + (collectionStates.size() + 1), slices, collectionProps, null); - collectionStates.put(docCollection.getName(), docCollection); - break; - case "s": - replicas = new HashMap<>(); - slice = new Slice("slice" + (slices.size() + 1), replicas, null); - slices.put(slice.getName(), slice); - break; - case "r": - Map replicaPropMap = new HashMap<>(); - String node; - - node = m.group(2); - - if (node == null || node.trim().length() == 0) { - node = "1"; - } - - Replica.State state = Replica.State.ACTIVE; - String stateCode = m.group(3); - - if (stateCode != null) { - switch (stateCode.charAt(0)) { - case 'S': - state = Replica.State.ACTIVE; - break; - case 'R': - state = Replica.State.RECOVERING; - break; - case 'D': - state = Replica.State.DOWN; - break; - case 'F': - state = Replica.State.RECOVERY_FAILED; - break; - default: - throw new IllegalArgumentException( - "Unexpected state for replica: " + stateCode); - } - } - - String nodeName = "baseUrl" + node + "_"; - String replicaName = "replica" + replicaCount++; - - if ("*".equals(m.group(4))) { - replicaName += " (bad)"; - } - - replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, nodeName); - replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node); - replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString()); - - replica = new Replica(replicaName, replicaPropMap); - - if ("*".equals(m.group(4))) { - result.badReplica = new DownReplica(); - result.badReplica.replica = replica; - result.badReplica.slice = slice; - result.badReplica.collection = docCollection; - } - - replicas.put(replica.getName(), replica); - break; - default: - break; - } - } - - // trunk briefly had clusterstate taking a zkreader :( this was required to work around that - leaving - // until that issue is resolved. - MockZkStateReader reader = new MockZkStateReader(null, collectionStates.keySet()); - ClusterState clusterState = new ClusterState(1, new HashSet<>(Arrays.asList(liveNodes)), collectionStates); - reader = new MockZkStateReader(clusterState, collectionStates.keySet()); - - String json; - try { - json = new String(Utils.toJSON(clusterState), "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unexpected"); - } - System.err.println("build:" + buildNumber++); - System.err.println(json); - - assert result.badReplica != null : "Is there no bad replica?"; - assert result.badReplica.slice != null : "Is there no bad replica?"; - - result.reader = reader; - - results.add(result); - - return result; - } }