Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12792: extract out test data

This commit is contained in:
Noble Paul 2018-10-01 22:00:09 +10:00
parent 93e2ae3fc4
commit 86e00405b0
8 changed files with 184 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
View File

@ -20,7 +20,6 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
@ -48,12 +47,12 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
import static java.util.Arrays.asList;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
@ -191,14 +190,14 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
String nodeName = ((NamedList) response.getResponse().get("success")).getName(0);
String corename = (String) ((NamedList) ((NamedList) response.getResponse().get("success")).getVal(0)).get("core");
String corename = (String) response.getResponse()._get(asList("success", nodeName,"core"),null);
try (HttpSolrClient coreclient = getHttpSolrClient(cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(nodeName))) {
CoreAdminResponse status = CoreAdminRequest.getStatus(corename, coreclient);
Map m = status.getResponse().asMap(5);
assertEquals(collectionName, Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "collection")));
assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "shard")));
assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "replica")));
NamedList m = status.getResponse();
assertEquals(collectionName, m._get(asList("status", corename, "cloud", "collection"), null));
assertNotNull(m._get(asList("status", corename, "cloud", "shard"), null));
assertNotNull(m._get(asList("status", corename, "cloud", "replica"), null));
}
}

9
solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
View File

@ -38,6 +38,15 @@ public interface IteratorWriter {
*/
ItemWriter add(Object o) throws IOException;
default ItemWriter addNoEx(Object o) {
try {
add(o);
} catch (IOException e) {
throw new RuntimeException(e);
}
return this;
}
default ItemWriter add(int v) throws IOException {
add((Integer) v);
return this;

20
solr/solrj/src/java/org/apache/solr/common/MapWriter.java
View File

@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
import org.apache.solr.common.util.Utils;
@ -90,6 +91,25 @@ public interface MapWriter extends MapSerializable {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : v;
}
default void _forEachEntry(String path, BiConsumer fun) {
Utils.forEachMapEntry(this, path, fun);
}
default void _forEachEntry(BiConsumer fun) {
Utils.forEachMapEntry(this, null, fun);
}
/**
* Get a child object value using json path
*
* @param path the full path to that object such as ["a","b","c[4]","d"] etc
* @param def the default
* @return the found value or default
*/
default Object _get(List<String> path, Object def) {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : v;
}
/**
* An interface to push one entry at a time to the output.
* The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once

10
solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.common.util;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@ -29,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -59,7 +61,7 @@ import org.apache.solr.common.params.SolrParams;
* </p>
*
*/
public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> {
public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> , MapWriter {
private static final long serialVersionUID = 1957981902839867821L;
protected final List<Object> nvPairs;
@ -74,6 +76,12 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
nvPairs = new ArrayList<>(sz<<1);
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
for (int i = 0; i < nvPairs.size(); i+=2) {
ew.put((String) nvPairs.get(i), nvPairs.get(i+1));
}
}
/**
* Creates a NamedList instance containing the "name,value" pairs contained in the

49
solr/solrj/src/java/org/apache/solr/common/util/Utils.java
View File

@ -43,6 +43,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -102,6 +103,26 @@ public class Utils {
return mutable ? copy : Collections.unmodifiableMap(copy);
}
public static void forEachMapEntry(MapWriter mw, String path, BiConsumer fun) {
Object o = path == null ? mw : Utils.getObjectByPath(mw, false, path);
if (o instanceof MapWriter) {
MapWriter m = (MapWriter) o;
try {
m.writeMap(new MapWriter.EntryWriter() {
@Override
public MapWriter.EntryWriter put(String k, Object v) {
fun.accept(k, v);
return this;
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
} else if (o instanceof Map) {
((Map) o).forEach((k, v) -> fun.accept(k, v));
}
}
private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable, boolean sorted) {
if (v instanceof MapWriter && maxDepth > 1) {
v = ((MapWriter) v).toMap(new LinkedHashMap<>());
@ -295,7 +316,7 @@ public class Utils {
}
}
if (i < hierarchy.size() - 1) {
Object o = getVal(obj, s);
Object o = getVal(obj, s, -1);
if (o == null) return false;
if (idx > -1) {
List l = (List) o;
@ -315,7 +336,7 @@ public class Utils {
}
return true;
} else {
Object v = getVal(obj, s);
Object v = getVal(obj, s, -1);
if (v instanceof List) {
List list = (List) v;
if (idx == -1) {
@ -352,7 +373,7 @@ public class Utils {
}
}
if (i < hierarchy.size() - 1) {
Object o = getVal(obj, s);
Object o = getVal(obj, s, -1);
if (o == null) return null;
if (idx > -1) {
List l = (List) o;
@ -361,10 +382,13 @@ public class Utils {
if (!isMapLike(o)) return null;
obj = o;
} else {
Object val = getVal(obj, s);
Object val = getVal(obj, s, -1);
if (val == null) return null;
if (idx > -1) {
if (val instanceof IteratorWriter) {
if (val instanceof MapWriter) {
val = getVal((MapWriter) val, null, idx);
} else if (val instanceof IteratorWriter) {
val = getValueAt((IteratorWriter) val, idx);
} else {
List l = (List) val;
@ -381,6 +405,7 @@ public class Utils {
return false;
}
private static Object getValueAt(IteratorWriter iteratorWriter, int idx) {
Object[] result = new Object[1];
try {
@ -406,14 +431,20 @@ public class Utils {
return o instanceof Map || o instanceof NamedList || o instanceof MapWriter;
}
private static Object getVal(Object obj, String key) {
private static Object getVal(Object obj, String key, int idx) {
if (obj instanceof MapWriter) {
Object[] result = new Object[1];
try {
((MapWriter) obj).writeMap(new MapWriter.EntryWriter() {
int count = -1;
@Override
public MapWriter.EntryWriter put(String k, Object v) throws IOException {
if (key.equals(k)) result[0] = v;
public MapWriter.EntryWriter put(String k, Object v) {
if (result[0] != null) return this;
if (k != null) {
if (key.equals(k)) result[0] = v;
} else {
if (++count == idx) result[0] = v;
}
return this;
}
});
@ -422,8 +453,6 @@ public class Utils {
}
return result[0];
}
if (obj instanceof NamedList) return ((NamedList) obj).get(key);
else if (obj instanceof Map) return ((Map) obj).get(key);
else throw new RuntimeException("must be a NamedList or Map");
}

88
solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json Normal file
View File

@ -0,0 +1,88 @@
{"collection1":{
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collection1_shard1_replica_n1",
"base_url":"http://127.0.0.1:51650/solr",
"node_name":"node1",
"state":"active",
"type":"NRT",
"leader":"true"},
"core_node6":{
"core":"collection1_shard1_replica_n3",
"base_url":"http://127.0.0.1:51651/solr",
"node_name":"node3",
"state":"active",
"type":"NRT"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"collection1_shard2_replica_n1",
"base_url":"http://127.0.0.1:51650/solr",
"node_name":"node1",
"state":"active",
"type":"NRT",
"leader":"true"},
"core_node5":{
"core":"collection1_shard2_replica_n3",
"base_url":"http://127.0.0.1:51651/solr",
"node_name":"node3",
"state":"active",
"type":"NRT"}}}},
"router":{
"name":"compositeId"},
"maxShardsPerNode":"2",
"autoAddReplicas":"true",
"nrtReplicas":"2",
"tlogReplicas":"0"},
"collection2":{
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{
"shard1":{
"range":"80000000-ffffffff",
"state":"active",
"replicas":{
"core_node1":{
"core":"collection2_shard1_replica_n1",
"base_url":"http://127.0.0.1:51649/solr",
"node_name":"node2",
"state":"active",
"type":"NRT"},
"core_node2":{
"core":"collection2_shard1_replica_n2",
"base_url":"http://127.0.0.1:51651/solr",
"node_name":"node3",
"state":"active",
"type":"NRT",
"leader":"true"}}},
"shard2":{
"range":"0-7fffffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"collection2_shard2_replica_n1",
"base_url":"http://127.0.0.1:51649/solr",
"node_name":"node2",
"state":"active",
"type":"NRT"},
"core_node4":{
"core":"collection2_shard2_replica_n2",
"base_url":"http://127.0.0.1:51651/solr",
"node_name":"node3",
"state":"active",
"type":"NRT",
"leader":"true"}}}},
"router":{
"name":"compositeId"},
"maxShardsPerNode":"2",
"autoAddReplicas":"true",
"nrtReplicas":"2",
"tlogReplicas":"0"}}

106
solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
View File

@ -143,9 +143,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'node_name':'node1'," +
" 'state':'active'}}}}}}";
public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, String clusterState) {
public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, Map clusterState) {
ValidatingJsonMap m = ValidatingJsonMap
.getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
.getDeepCopy(clusterState, 6, true);
Map<String, Map<String, List<ReplicaInfo>>> result = new LinkedHashMap<>();
m.forEach((collName, o) -> {
@ -1426,101 +1426,14 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
public void testMoveReplicasInMultipleCollections() {
public void testMoveReplicasInMultipleCollections() throws IOException {
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:2}," +
"node3:{cores:4}" +
"}");
String clusterState = "{\n" +
"'collection1' : {\n" +
" 'pullReplicas':'0',\n" +
" 'replicationFactor':'2',\n" +
" 'shards':{\n" +
" 'shard1':{\n" +
" 'range':'80000000-ffffffff',\n" +
" 'state':'active',\n" +
" 'replicas':{\n" +
" 'core_node1':{\n" +
" 'core':'collection1_shard1_replica_n1',\n" +
" 'base_url':'http://127.0.0.1:51650/solr',\n" +
" 'node_name':'node1',\n" +
" 'state':'active',\n" +
" 'type':'NRT',\n" +
" 'leader':'true'},\n" +
" 'core_node6':{\n" +
" 'core':'collection1_shard1_replica_n3',\n" +
" 'base_url':'http://127.0.0.1:51651/solr',\n" +
" 'node_name':'node3',\n" +
" 'state':'active',\n" +
" 'type':'NRT'}}},\n" +
" 'shard2':{\n" +
" 'range':'0-7fffffff',\n" +
" 'state':'active',\n" +
" 'replicas':{\n" +
" 'core_node3':{\n" +
" 'core':'collection1_shard2_replica_n1',\n" +
" 'base_url':'http://127.0.0.1:51650/solr',\n" +
" 'node_name':'node1',\n" +
" 'state':'active',\n" +
" 'type':'NRT',\n" +
" 'leader':'true'},\n" +
" 'core_node5':{\n" +
" 'core':'collection1_shard2_replica_n3',\n" +
" 'base_url':'http://127.0.0.1:51651/solr',\n" +
" 'node_name':'node3',\n" +
" 'state':'active',\n" +
" 'type':'NRT'}}}},\n" +
" 'router':{'name':'compositeId'},\n" +
" 'maxShardsPerNode':'2',\n" +
" 'autoAddReplicas':'true',\n" +
" 'nrtReplicas':'2',\n" +
" 'tlogReplicas':'0'},\n" +
"'collection2' : {\n" +
" 'pullReplicas':'0',\n" +
" 'replicationFactor':'2',\n" +
" 'shards':{\n" +
" 'shard1':{\n" +
" 'range':'80000000-ffffffff',\n" +
" 'state':'active',\n" +
" 'replicas':{\n" +
" 'core_node1':{\n" +
" 'core':'collection2_shard1_replica_n1',\n" +
" 'base_url':'http://127.0.0.1:51649/solr',\n" +
" 'node_name':'node2',\n" +
" 'state':'active',\n" +
" 'type':'NRT'},\n" +
" 'core_node2':{\n" +
" 'core':'collection2_shard1_replica_n2',\n" +
" 'base_url':'http://127.0.0.1:51651/solr',\n" +
" 'node_name':'node3',\n" +
" 'state':'active',\n" +
" 'type':'NRT',\n" +
" 'leader':'true'}}},\n" +
" 'shard2':{\n" +
" 'range':'0-7fffffff',\n" +
" 'state':'active',\n" +
" 'replicas':{\n" +
" 'core_node3':{\n" +
" 'core':'collection2_shard2_replica_n1',\n" +
" 'base_url':'http://127.0.0.1:51649/solr',\n" +
" 'node_name':'node2',\n" +
" 'state':'active',\n" +
" 'type':'NRT'},\n" +
" 'core_node4':{\n" +
" 'core':'collection2_shard2_replica_n2',\n" +
" 'base_url':'http://127.0.0.1:51651/solr',\n" +
" 'node_name':'node3',\n" +
" 'state':'active',\n" +
" 'type':'NRT',\n" +
" 'leader':'true'}}}},\n" +
" 'router':{'name':'compositeId'},\n" +
" 'maxShardsPerNode':'2',\n" +
" 'autoAddReplicas':'true',\n" +
" 'nrtReplicas':'2',\n" +
" 'tlogReplicas':'0'}\n" +
"}";
Policy policy = new Policy(new HashMap<>());
Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState))
Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues,
(Map<String, Object>) TestPolicy2.loadFromResource("testMoveReplicasInMultipleCollections.json")))
.getSuggester(MOVEREPLICA)
.hint(Hint.COLL, "collection1")
.hint(Hint.COLL, "collection2")
@ -2083,8 +1996,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(op);
assertEquals("node2", op.getNode());
}
static SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, String clusterS) {
return getSolrCloudManager(nodeValues,(Map) Utils.fromJSONString(clusterS));
private SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, String clusterS) {
}
private static SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, Map clusterS) {
return new SolrCloudManager() {
ObjectCache objectCache = new ObjectCache();
@ -2188,7 +2104,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
@Override
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState));
}
};
}
@ -2240,7 +2156,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
@Override
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState));
}
};
}

6
solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
View File

@ -201,9 +201,9 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson)),
createCloudManager(state, metaData));
assertEquals(1, suggestions.size());
assertEquals("node5", Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/targetNode"));
assertEquals("node5", suggestions.get(0)._get("operation/command/move-replica/targetNode", null));
String rName = (String) Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/replica");
String rName = (String) suggestions.get(0)._get("operation/command/move-replica/replica", null);
found.set(false);
session.getNode("node1").forEachReplica(replicaInfo -> {
@ -412,8 +412,6 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null));
}
}
public static Object loadFromResource(String file) throws IOException {