diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index f61fa5ef173..5390704c247 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -48,12 +47,12 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.junit.BeforeClass; import org.junit.Test; +import static java.util.Arrays.asList; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP; @@ -191,14 +190,14 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase { assertEquals(0, response.getStatus()); assertTrue(response.isSuccess()); String nodeName = ((NamedList) response.getResponse().get("success")).getName(0); - String corename = (String) ((NamedList) ((NamedList) response.getResponse().get("success")).getVal(0)).get("core"); + String corename = (String) response.getResponse()._get(asList("success", nodeName,"core"),null); try (HttpSolrClient coreclient = getHttpSolrClient(cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(nodeName))) { CoreAdminResponse status = CoreAdminRequest.getStatus(corename, coreclient); - Map m = status.getResponse().asMap(5); - assertEquals(collectionName, Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "collection"))); - assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "shard"))); - assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "replica"))); + NamedList m = status.getResponse(); + assertEquals(collectionName, m._get(asList("status", corename, "cloud", "collection"), null)); + assertNotNull(m._get(asList("status", corename, "cloud", "shard"), null)); + assertNotNull(m._get(asList("status", corename, "cloud", "replica"), null)); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java index cbfb58473ff..ec11c786d02 100644 --- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java @@ -38,6 +38,15 @@ public interface IteratorWriter { */ ItemWriter add(Object o) throws IOException; + default ItemWriter addNoEx(Object o) { + try { + add(o); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + default ItemWriter add(int v) throws IOException { add((Integer) v); return this; diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java index 9b1861a842d..3ec37e7a526 100644 --- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.BiPredicate; import org.apache.solr.common.util.Utils; @@ -90,6 +91,25 @@ public interface MapWriter extends MapSerializable { Object v = Utils.getObjectByPath(this, false, path); return v == null ? def : v; } + default void _forEachEntry(String path, BiConsumer fun) { + Utils.forEachMapEntry(this, path, fun); + } + + default void _forEachEntry(BiConsumer fun) { + Utils.forEachMapEntry(this, null, fun); + } + + /** + * Get a child object value using json path + * + * @param path the full path to that object such as ["a","b","c[4]","d"] etc + * @param def the default + * @return the found value or default + */ + default Object _get(List path, Object def) { + Object v = Utils.getObjectByPath(this, false, path); + return v == null ? def : v; + } /** * An interface to push one entry at a time to the output. * The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java index 16506028780..71cb78b5072 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java @@ -16,6 +16,7 @@ */ package org.apache.solr.common.util; +import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -29,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.params.SolrParams; @@ -59,7 +61,7 @@ import org.apache.solr.common.params.SolrParams; *

* */ -public class NamedList implements Cloneable, Serializable, Iterable> { +public class NamedList implements Cloneable, Serializable, Iterable> , MapWriter { private static final long serialVersionUID = 1957981902839867821L; protected final List nvPairs; @@ -74,6 +76,12 @@ public class NamedList implements Cloneable, Serializable, Iterable(sz<<1); } + @Override + public void writeMap(EntryWriter ew) throws IOException { + for (int i = 0; i < nvPairs.size(); i+=2) { + ew.put((String) nvPairs.get(i), nvPairs.get(i+1)); + } + } /** * Creates a NamedList instance containing the "name,value" pairs contained in the diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 800c2c125a0..2d1df428036 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -43,6 +43,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -102,6 +103,26 @@ public class Utils { return mutable ? copy : Collections.unmodifiableMap(copy); } + public static void forEachMapEntry(MapWriter mw, String path, BiConsumer fun) { + Object o = path == null ? mw : Utils.getObjectByPath(mw, false, path); + if (o instanceof MapWriter) { + MapWriter m = (MapWriter) o; + try { + m.writeMap(new MapWriter.EntryWriter() { + @Override + public MapWriter.EntryWriter put(String k, Object v) { + fun.accept(k, v); + return this; + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else if (o instanceof Map) { + ((Map) o).forEach((k, v) -> fun.accept(k, v)); + } + } + private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable, boolean sorted) { if (v instanceof MapWriter && maxDepth > 1) { v = ((MapWriter) v).toMap(new LinkedHashMap<>()); @@ -295,7 +316,7 @@ public class Utils { } } if (i < hierarchy.size() - 1) { - Object o = getVal(obj, s); + Object o = getVal(obj, s, -1); if (o == null) return false; if (idx > -1) { List l = (List) o; @@ -315,7 +336,7 @@ public class Utils { } return true; } else { - Object v = getVal(obj, s); + Object v = getVal(obj, s, -1); if (v instanceof List) { List list = (List) v; if (idx == -1) { @@ -352,7 +373,7 @@ public class Utils { } } if (i < hierarchy.size() - 1) { - Object o = getVal(obj, s); + Object o = getVal(obj, s, -1); if (o == null) return null; if (idx > -1) { List l = (List) o; @@ -361,10 +382,13 @@ public class Utils { if (!isMapLike(o)) return null; obj = o; } else { - Object val = getVal(obj, s); + Object val = getVal(obj, s, -1); if (val == null) return null; if (idx > -1) { - if (val instanceof IteratorWriter) { + if (val instanceof MapWriter) { + val = getVal((MapWriter) val, null, idx); + + } else if (val instanceof IteratorWriter) { val = getValueAt((IteratorWriter) val, idx); } else { List l = (List) val; @@ -381,6 +405,7 @@ public class Utils { return false; } + private static Object getValueAt(IteratorWriter iteratorWriter, int idx) { Object[] result = new Object[1]; try { @@ -406,14 +431,20 @@ public class Utils { return o instanceof Map || o instanceof NamedList || o instanceof MapWriter; } - private static Object getVal(Object obj, String key) { + private static Object getVal(Object obj, String key, int idx) { if (obj instanceof MapWriter) { Object[] result = new Object[1]; try { ((MapWriter) obj).writeMap(new MapWriter.EntryWriter() { + int count = -1; @Override - public MapWriter.EntryWriter put(String k, Object v) throws IOException { - if (key.equals(k)) result[0] = v; + public MapWriter.EntryWriter put(String k, Object v) { + if (result[0] != null) return this; + if (k != null) { + if (key.equals(k)) result[0] = v; + } else { + if (++count == idx) result[0] = v; + } return this; } }); @@ -422,8 +453,6 @@ public class Utils { } return result[0]; } - - if (obj instanceof NamedList) return ((NamedList) obj).get(key); else if (obj instanceof Map) return ((Map) obj).get(key); else throw new RuntimeException("must be a NamedList or Map"); } diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json new file mode 100644 index 00000000000..16ba1a77ec3 --- /dev/null +++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json @@ -0,0 +1,88 @@ +{"collection1":{ + "pullReplicas":"0", + "replicationFactor":"2", + "shards":{ + "shard1":{ + "range":"80000000-ffffffff", + "state":"active", + "replicas":{ + "core_node1":{ + "core":"collection1_shard1_replica_n1", + "base_url":"http://127.0.0.1:51650/solr", + "node_name":"node1", + "state":"active", + "type":"NRT", + "leader":"true"}, + "core_node6":{ + "core":"collection1_shard1_replica_n3", + "base_url":"http://127.0.0.1:51651/solr", + "node_name":"node3", + "state":"active", + "type":"NRT"}}}, + "shard2":{ + "range":"0-7fffffff", + "state":"active", + "replicas":{ + "core_node3":{ + "core":"collection1_shard2_replica_n1", + "base_url":"http://127.0.0.1:51650/solr", + "node_name":"node1", + "state":"active", + "type":"NRT", + "leader":"true"}, + "core_node5":{ + "core":"collection1_shard2_replica_n3", + "base_url":"http://127.0.0.1:51651/solr", + "node_name":"node3", + "state":"active", + "type":"NRT"}}}}, + "router":{ + "name":"compositeId"}, + "maxShardsPerNode":"2", + "autoAddReplicas":"true", + "nrtReplicas":"2", + "tlogReplicas":"0"}, + "collection2":{ + "pullReplicas":"0", + "replicationFactor":"2", + "shards":{ + "shard1":{ + "range":"80000000-ffffffff", + "state":"active", + "replicas":{ + "core_node1":{ + "core":"collection2_shard1_replica_n1", + "base_url":"http://127.0.0.1:51649/solr", + "node_name":"node2", + "state":"active", + "type":"NRT"}, + "core_node2":{ + "core":"collection2_shard1_replica_n2", + "base_url":"http://127.0.0.1:51651/solr", + "node_name":"node3", + "state":"active", + "type":"NRT", + "leader":"true"}}}, + "shard2":{ + "range":"0-7fffffff", + "state":"active", + "replicas":{ + "core_node3":{ + "core":"collection2_shard2_replica_n1", + "base_url":"http://127.0.0.1:51649/solr", + "node_name":"node2", + "state":"active", + "type":"NRT"}, + "core_node4":{ + "core":"collection2_shard2_replica_n2", + "base_url":"http://127.0.0.1:51651/solr", + "node_name":"node3", + "state":"active", + "type":"NRT", + "leader":"true"}}}}, + "router":{ + "name":"compositeId"}, + "maxShardsPerNode":"2", + "autoAddReplicas":"true", + "nrtReplicas":"2", + "tlogReplicas":"0"}} \ No newline at end of file diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java index 9c5be7f8c7c..29e45ed85c1 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java @@ -143,9 +143,9 @@ public class TestPolicy extends SolrTestCaseJ4 { " 'node_name':'node1'," + " 'state':'active'}}}}}}"; - public static Map>> getReplicaDetails(String node, String clusterState) { + public static Map>> getReplicaDetails(String node, Map clusterState) { ValidatingJsonMap m = ValidatingJsonMap - .getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true); + .getDeepCopy(clusterState, 6, true); Map>> result = new LinkedHashMap<>(); m.forEach((collName, o) -> { @@ -1426,101 +1426,14 @@ public class TestPolicy extends SolrTestCaseJ4 { } - public void testMoveReplicasInMultipleCollections() { + public void testMoveReplicasInMultipleCollections() throws IOException { Map nodeValues = (Map) Utils.fromJSONString("{" + "node1:{cores:2}," + "node3:{cores:4}" + "}"); - String clusterState = "{\n" + - "'collection1' : {\n" + - " 'pullReplicas':'0',\n" + - " 'replicationFactor':'2',\n" + - " 'shards':{\n" + - " 'shard1':{\n" + - " 'range':'80000000-ffffffff',\n" + - " 'state':'active',\n" + - " 'replicas':{\n" + - " 'core_node1':{\n" + - " 'core':'collection1_shard1_replica_n1',\n" + - " 'base_url':'http://127.0.0.1:51650/solr',\n" + - " 'node_name':'node1',\n" + - " 'state':'active',\n" + - " 'type':'NRT',\n" + - " 'leader':'true'},\n" + - " 'core_node6':{\n" + - " 'core':'collection1_shard1_replica_n3',\n" + - " 'base_url':'http://127.0.0.1:51651/solr',\n" + - " 'node_name':'node3',\n" + - " 'state':'active',\n" + - " 'type':'NRT'}}},\n" + - " 'shard2':{\n" + - " 'range':'0-7fffffff',\n" + - " 'state':'active',\n" + - " 'replicas':{\n" + - " 'core_node3':{\n" + - " 'core':'collection1_shard2_replica_n1',\n" + - " 'base_url':'http://127.0.0.1:51650/solr',\n" + - " 'node_name':'node1',\n" + - " 'state':'active',\n" + - " 'type':'NRT',\n" + - " 'leader':'true'},\n" + - " 'core_node5':{\n" + - " 'core':'collection1_shard2_replica_n3',\n" + - " 'base_url':'http://127.0.0.1:51651/solr',\n" + - " 'node_name':'node3',\n" + - " 'state':'active',\n" + - " 'type':'NRT'}}}},\n" + - " 'router':{'name':'compositeId'},\n" + - " 'maxShardsPerNode':'2',\n" + - " 'autoAddReplicas':'true',\n" + - " 'nrtReplicas':'2',\n" + - " 'tlogReplicas':'0'},\n" + - "'collection2' : {\n" + - " 'pullReplicas':'0',\n" + - " 'replicationFactor':'2',\n" + - " 'shards':{\n" + - " 'shard1':{\n" + - " 'range':'80000000-ffffffff',\n" + - " 'state':'active',\n" + - " 'replicas':{\n" + - " 'core_node1':{\n" + - " 'core':'collection2_shard1_replica_n1',\n" + - " 'base_url':'http://127.0.0.1:51649/solr',\n" + - " 'node_name':'node2',\n" + - " 'state':'active',\n" + - " 'type':'NRT'},\n" + - " 'core_node2':{\n" + - " 'core':'collection2_shard1_replica_n2',\n" + - " 'base_url':'http://127.0.0.1:51651/solr',\n" + - " 'node_name':'node3',\n" + - " 'state':'active',\n" + - " 'type':'NRT',\n" + - " 'leader':'true'}}},\n" + - " 'shard2':{\n" + - " 'range':'0-7fffffff',\n" + - " 'state':'active',\n" + - " 'replicas':{\n" + - " 'core_node3':{\n" + - " 'core':'collection2_shard2_replica_n1',\n" + - " 'base_url':'http://127.0.0.1:51649/solr',\n" + - " 'node_name':'node2',\n" + - " 'state':'active',\n" + - " 'type':'NRT'},\n" + - " 'core_node4':{\n" + - " 'core':'collection2_shard2_replica_n2',\n" + - " 'base_url':'http://127.0.0.1:51651/solr',\n" + - " 'node_name':'node3',\n" + - " 'state':'active',\n" + - " 'type':'NRT',\n" + - " 'leader':'true'}}}},\n" + - " 'router':{'name':'compositeId'},\n" + - " 'maxShardsPerNode':'2',\n" + - " 'autoAddReplicas':'true',\n" + - " 'nrtReplicas':'2',\n" + - " 'tlogReplicas':'0'}\n" + - "}"; Policy policy = new Policy(new HashMap<>()); - Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState)) + Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, + (Map) TestPolicy2.loadFromResource("testMoveReplicasInMultipleCollections.json"))) .getSuggester(MOVEREPLICA) .hint(Hint.COLL, "collection1") .hint(Hint.COLL, "collection2") @@ -2083,8 +1996,11 @@ public class TestPolicy extends SolrTestCaseJ4 { assertNotNull(op); assertEquals("node2", op.getNode()); } + static SolrCloudManager getSolrCloudManager(final Map nodeValues, String clusterS) { + return getSolrCloudManager(nodeValues,(Map) Utils.fromJSONString(clusterS)); - private SolrCloudManager getSolrCloudManager(final Map nodeValues, String clusterS) { + } + private static SolrCloudManager getSolrCloudManager(final Map nodeValues, Map clusterS) { return new SolrCloudManager() { ObjectCache objectCache = new ObjectCache(); @@ -2188,7 +2104,7 @@ public class TestPolicy extends SolrTestCaseJ4 { @Override public Map>> getReplicaInfo(String node, Collection keys) { - return getReplicaDetails(node, clusterState); + return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState)); } }; } @@ -2240,7 +2156,7 @@ public class TestPolicy extends SolrTestCaseJ4 { @Override public Map>> getReplicaInfo(String node, Collection keys) { - return getReplicaDetails(node, clusterState); + return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState)); } }; } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java index 615fd4db806..07ce391237f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java @@ -201,9 +201,9 @@ public class TestPolicy2 extends SolrTestCaseJ4 { suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)), createCloudManager(state, metaData)); assertEquals(1, suggestions.size()); - assertEquals("node5", Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/targetNode")); + assertEquals("node5", suggestions.get(0)._get("operation/command/move-replica/targetNode", null)); - String rName = (String) Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/replica"); + String rName = (String) suggestions.get(0)._get("operation/command/move-replica/replica", null); found.set(false); session.getNode("node1").forEachReplica(replicaInfo -> { @@ -412,8 +412,6 @@ public class TestPolicy2 extends SolrTestCaseJ4 { assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null)); } - - } public static Object loadFromResource(String file) throws IOException {