Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10278: Implemented conditions

This commit is contained in:
Noble Paul 2017-03-20 23:01:03 +10:30
parent 541469aae3
commit a025f8bdb9
9 changed files with 473 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
View File

@ -28,7 +28,7 @@ import java.util.List;
*/
public interface IteratorWriter {
/**
* @param iw after this method returns , the EntryWriter Object is invalid
* @param iw after this method returns , the ItemWriter Object is invalid
* Do not hold a reference to this object
*/
void writeIter(ItemWriter iw) throws IOException;
@ -65,16 +65,20 @@ public interface IteratorWriter {
return this;
}
}
default List toList( List l) throws IOException {
writeIter(new IteratorWriter.ItemWriter() {
@Override
public IteratorWriter.ItemWriter add(Object o) throws IOException {
if (o instanceof MapWriter) o = ((MapWriter) o).toMap(new LinkedHashMap<>());
if (o instanceof IteratorWriter) o = ((IteratorWriter) o).toList(new ArrayList<>());
l.add(o);
return this;
}
});
default List toList( List l) {
try {
writeIter(new ItemWriter() {
@Override
public ItemWriter add(Object o) throws IOException {
if (o instanceof MapWriter) o = ((MapWriter) o).toMap(new LinkedHashMap<>());
if (o instanceof IteratorWriter) o = ((IteratorWriter) o).toList(new ArrayList<>());
l.add(o);
return this;
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
return l;
}
}

1
solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
View File

@ -94,6 +94,7 @@ public interface CollectionParams {
CREATESNAPSHOT(true, LockLevel.COLLECTION),
DELETESNAPSHOT(true, LockLevel.COLLECTION),
LISTSNAPSHOTS(false, LockLevel.NONE),
MOVEREPLICA(false, LockLevel.SHARD),
//only for testing. it just waits for specified time
// these are not exposed via collection API commands
// but the overseer is aware of these tasks

15
solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
View File

@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
@ -390,7 +392,18 @@ public class JavaBinCodec implements PushWriter {
writeMap(((MapSerializable) val).toMap(new NamedList().asShallowMap()));
return true;
}
if (val instanceof AtomicInteger) {
writeInt(((AtomicInteger) val).get());
return true;
}
if (val instanceof AtomicLong) {
writeLong(((AtomicLong) val).get());
return true;
}
if (val instanceof AtomicBoolean) {
writeBoolean(((AtomicBoolean) val).get());
return true;
}
return false;
}

22
solr/solrj/src/java/org/apache/solr/common/util/Utils.java
View File

@ -36,6 +36,8 @@ import java.util.regex.Pattern;
import org.apache.http.HttpEntity;
import org.apache.http.util.EntityUtils;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.noggit.CharArr;
import org.noggit.JSONParser;
@ -60,23 +62,23 @@ public class Utils {
Map copy = new LinkedHashMap();
for (Object o : map.entrySet()) {
Map.Entry e = (Map.Entry) o;
Object v = e.getValue();
if (v instanceof Map) v = getDeepCopy((Map) v, maxDepth - 1, mutable);
else if (v instanceof Collection) v = getDeepCopy((Collection) v, maxDepth - 1, mutable);
copy.put(e.getKey(), v);
copy.put(e.getKey(), makeDeepCopy(e.getValue(),maxDepth, mutable));
}
return mutable ? copy : Collections.unmodifiableMap(copy);
}
private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable) {
if (v instanceof MapWriter) v = ((MapWriter) v).toMap(new LinkedHashMap<>());
else if (v instanceof IteratorWriter) v = ((IteratorWriter) v).toList(new ArrayList<>());
else if (v instanceof Map) v = getDeepCopy((Map) v, maxDepth - 1, mutable);
else if (v instanceof Collection) v = getDeepCopy((Collection) v, maxDepth - 1, mutable);
return v;
}
public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable) {
if (c == null || maxDepth < 1) return c;
Collection result = c instanceof Set ? new HashSet() : new ArrayList();
for (Object o : c) {
if (o instanceof Map) {
o = getDeepCopy((Map) o, maxDepth - 1, mutable);
}
result.add(o);
}
for (Object o : c) result.add(makeDeepCopy(o, maxDepth, mutable));
return mutable ? result : result instanceof Set ? unmodifiableSet((Set) result) : unmodifiableList((List) result);
}

145
solr/solrj/src/java/org/apache/solr/recipe/Clause.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.recipe;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.recipe.RuleSorter.Row;
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
import static org.apache.solr.common.params.CoreAdminParams.SHARD;
import static org.apache.solr.recipe.Operand.EQUAL;
import static org.apache.solr.recipe.Operand.GREATER_THAN;
import static org.apache.solr.recipe.Operand.LESS_THAN;
import static org.apache.solr.recipe.Operand.NOT_EQUAL;
import static org.apache.solr.recipe.RuleSorter.ANY;
class Clause implements MapWriter {
Map<String, Object> original;
Condition collection, shard, replica, tag;
boolean strict = true;
Clause(Map<String, Object> m) {
this.original = m;
collection = new Condition(COLLECTION, m.containsKey(COLLECTION) ? (String) m.get(COLLECTION) : ANY, EQUAL);
shard = new Condition(SHARD, m.containsKey(SHARD) ? (String) m.get(SHARD) : ANY, EQUAL);
String replica = m.containsKey(REPLICA) ? String.valueOf(m.get(REPLICA)) : ANY;
this.replica = parse(REPLICA, replica);
strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
m.forEach(this::parseCondition);
if (tag == null)
throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
}
void parseCondition(String s, Object o) {
if (IGNORE_TAGS.contains(s)) return;
if (tag != null) {
throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
}
tag = parse(s, o);
}
class Condition {
final String name;
final Object val;
final Operand op;
Condition(String name, Object val, Operand op) {
this.name = name;
this.val = val;
this.op = op;
}
boolean isMatch(Object inputVal) {
return op.canMatch(val, inputVal);
}
}
Condition parse(String s, Object o) {
Object expectedVal;
String value = null;
try {
String conditionName = s.trim();
value = String.valueOf(o).trim();
Operand operand = null;
if ((expectedVal = NOT_EQUAL.match(value)) != null) {
operand = NOT_EQUAL;
} else if ((expectedVal = GREATER_THAN.match(value)) != null) {
operand = GREATER_THAN;
} else if ((expectedVal = LESS_THAN.match(value)) != null) {
operand = LESS_THAN;
} else {
operand = EQUAL;
expectedVal = EQUAL.match(value);
}
return new Condition(conditionName, expectedVal, operand);
} catch (Exception e) {
throw new IllegalArgumentException("Invalid tag : " + s + ":" + value, e);
}
}
TestStatus test(Row row) {
AtomicReference<TestStatus> result = new AtomicReference<>(TestStatus.NOT_APPLICABLE);
Object val = row.getVal(tag.name);
if (tag.isMatch(val)) {
checkReplicaCount(row, result);
if (result.get() == TestStatus.FAIL) row.violations.add(this);
}
return result.get();
}
TestStatus checkReplicaCount(Row row, AtomicReference<TestStatus> result) {
row.replicaInfo.forEach((coll, e) -> {
if (!collection.isMatch(coll)) return;
AtomicInteger count = new AtomicInteger();
e.forEach((sh, replicas) -> {
if (!shard.isMatch(sh)) return;
count.addAndGet(replicas.size());
});
result.set(replica.isMatch(count) ? TestStatus.PASS : TestStatus.FAIL);
if (RuleSorter.EACH.equals(shard.val)) count.set(0);
});
return TestStatus.PASS;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
}
enum TestStatus {
NOT_APPLICABLE, FAIL, PASS
}
private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
}

4
solr/solrj/src/java/org/apache/solr/recipe/Operand.java
View File

@ -19,9 +19,7 @@ package org.apache.solr.recipe;
import java.util.Objects;
/**
* Created by noble on 3/6/17.
*/
public enum Operand {
EQUAL(""),
NOT_EQUAL("!") {

62
solr/solrj/src/java/org/apache/solr/recipe/Preference.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.recipe;
import java.util.List;
import java.util.Map;
class Preference {
final RuleSorter.SortParam name;
Integer precision;
final RuleSorter.Sort sort;
Preference next;
public int idx;
Preference(Map<String, Object> m) {
sort = RuleSorter.Sort.get(m);
name = RuleSorter.SortParam.get(m.get(sort.name()).toString());
Object p = m.getOrDefault("precision", 0);
precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
}
// there are 2 modes of compare.
// recursive, it uses the precision to tie & when there is a tie use the next preference to compare
// in non-recursive mode, precision is not taken into consideration and sort is done on actual value
int compare(RuleSorter.Row r1, RuleSorter.Row r2, boolean recursive) {
Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val;
Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val;
int result = 0;
if (o1 instanceof Integer && o2 instanceof Integer) result = ((Integer) o1).compareTo((Integer) o2);
if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
if (o1 instanceof Float && o2 instanceof Float) result = ((Float) o1).compareTo((Float) o2);
if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
return result == 0 ? next == null ? 0 : next.compare(r1, r2, recursive) : sort.sortval * result;
}
//sets the new value according to precision in val_
void setApproxVal(List<RuleSorter.Row> tmpMatrix) {
Object prevVal = null;
for (RuleSorter.Row row : tmpMatrix) {
prevVal = row.cells[idx].val_ =
prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ?
row.cells[idx].val :
prevVal;
}
}
}

261
solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java
View File

@ -20,34 +20,40 @@ package org.apache.solr.recipe;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.util.Utils;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.recipe.Operand.GREATER_THAN;
import static org.apache.solr.recipe.Operand.LESS_THAN;
import static org.apache.solr.recipe.Operand.NOT_EQUAL;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
public class RuleSorter {
public static final String ALL = "#ALL";
public static final String EACH = "#EACH";
List<Clause> conditionClauses = new ArrayList<>();
public static final String ANY = "#ANY";
List<Clause> clauses = new ArrayList<>();
List<Preference> preferences = new ArrayList<>();
List<String> params= new ArrayList<>();
public RuleSorter(Map<String, Object> jsonMap) {
List<Map<String, Object>> l = getListOfMap("conditions", jsonMap);
conditionClauses = l.stream().map(Clause::new).collect(toList());
clauses = l.stream().map(Clause::new).collect(toList());
l = getListOfMap("preferences", jsonMap);
preferences = l.stream().map(Preference::new).collect(toList());
for (int i = 0; i < preferences.size() - 1; i++) {
@ -55,10 +61,7 @@ public class RuleSorter {
preference.next = preferences.get(i + 1);
}
for (Clause c : conditionClauses) {
for (Condition condition : c.conditions) params.add(condition.name);
}
for (Clause c : clauses) params.add(c.tag.name);
for (Preference preference : preferences) {
if (params.contains(preference.name.name())) {
throw new RuntimeException(preference.name + " is repeated");
@ -66,34 +69,64 @@ public class RuleSorter {
params.add(preference.name.toString());
preference.idx = params.size() - 1;
}
}
public class Session implements MapWriter {
private final List<String> nodes;
private final NodeValueProvider snitch;
private List<Row> matrix;
List<String> paramsList = new ArrayList<>(params);
private Session(List<String> nodes, NodeValueProvider snitch) {
public class Session implements MapWriter {
final List<String> nodes;
final NodeValueProvider snitch;
final List<Row> matrix;
Set<String> collections = new HashSet<>();
Session(List<String> nodes, NodeValueProvider snitch) {
this.nodes = nodes;
this.snitch = snitch;
matrix = new ArrayList<>(nodes.size());
for (String node : nodes) matrix.add(new Row(node, paramsList, snitch));
for (String node : nodes) matrix.add(new Row(node, params, snitch));
for (Row row : matrix) row.replicaInfo.forEach((s, e) -> collections.add(s));
}
public void sort() {
if (preferences.size() > 1) {
List<Row> getMatrixCopy() {
return matrix.stream()
.map(Row::copy)
.collect(Collectors.toList());
}
/**Apply the preferences and conditions
*/
public void applyRules() {
if (!preferences.isEmpty()) {
//this is to set the approximate value according to the precision
ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
for (Preference p : preferences) {
Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
p.setNewVal(tmpMatrix);
p.setApproxVal(tmpMatrix);
}
//approximate values are set now. Let's do recursive sorting
Collections.sort(matrix, (r1, r2) -> preferences.get(0).compare(r1, r2, true));
}
if (!clauses.isEmpty()) {
for (Clause clause : clauses) {
for (Row row : matrix) {
clause.test(row);
}
}
}
}
public Map<String, List<Clause>> getViolations() {
return matrix.stream()
.filter(row -> !row.violations.isEmpty())
.collect(Collectors.toMap(r -> r.node, r -> r.violations));
}
public Operation suggest(CollectionAction action) {
if (!supportedActions.contains(action))
throw new UnsupportedOperationException(action.toString() + "is not supported");
return null;
}
@Override
@ -130,103 +163,6 @@ public class RuleSorter {
}
static class Clause {
List<Condition> conditions;
boolean strict = true;
Clause(Map<String, Object> m) {
conditions = m.entrySet().stream()
.filter(e -> !"strict".equals(e.getKey().trim()))
.map(Condition::new)
.collect(toList());
Object o = m.get("strict");
if (o == null) return;
strict = o instanceof Boolean ? (Boolean) o : Boolean.parseBoolean(o.toString());
}
}
static class Condition {
String name;
Object val;
Operand operand;
Condition(Map.Entry<String, Object> m) {
Object expectedVal;
try {
this.name = m.getKey().trim();
String value = m.getValue().toString().trim();
if ((expectedVal = NOT_EQUAL.match(value)) != null) {
operand = NOT_EQUAL;
} else if ((expectedVal = GREATER_THAN.match(value)) != null) {
operand = GREATER_THAN;
} else if ((expectedVal = LESS_THAN.match(value)) != null) {
operand = LESS_THAN;
} else {
operand = Operand.EQUAL;
expectedVal = value;
}
if (name.equals(REPLICA_PROP)) {
if (!ALL.equals(expectedVal)) {
try {
expectedVal = Integer.parseInt(expectedVal.toString());
} catch (NumberFormatException e) {
throw new RuntimeException("The replica tag value can only be '*' or an integer");
}
}
}
} catch (Exception e) {
throw new IllegalArgumentException("Invalid condition : " + name + ":" + val, e);
}
this.val = expectedVal;
}
}
static class Preference {
final SortParam name;
Integer precision;
final Sort sort;
Preference next;
public int idx;
Preference(Map<String, Object> m) {
sort = Sort.get(m);
name = SortParam.get(m.get(sort.name()).toString());
Object p = m.getOrDefault("precision", 0);
precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
}
// there are 2 modes of compare.
// recursive, it uses the precision to tie & when there is a tie use the next preference to compare
// in non-recursive mode, precision is not taken into consideration and sort is done on actual value
int compare(Row r1, Row r2, boolean recursive) {
Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val;
Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val;
int result = 0;
if (o1 instanceof Integer && o2 instanceof Integer) result = ((Integer) o1).compareTo((Integer) o2);
if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
if (o1 instanceof Float && o2 instanceof Float) result = ((Float) o1).compareTo((Float) o2);
if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
return result == 0 ? next == null ? 0 : next.compare(r1, r2, recursive) : sort.sortval * result;
}
//sets the new value according to precision in val_
void setNewVal(List<Row> tmpMatrix) {
Object prevVal = null;
for (Row row : tmpMatrix) {
prevVal = row.cells[idx].val_ =
prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ?
row.cells[idx].val :
prevVal;
}
}
}
enum SortParam {
freedisk, cores, heap, cpu;
@ -245,7 +181,7 @@ public class RuleSorter {
sortval = i;
}
public static Sort get(Map<String, Object> m) {
static Sort get(Map<String, Object> m) {
if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
}
@ -255,30 +191,54 @@ public class RuleSorter {
}
}
public static class Row implements MapWriter {
static class Row implements MapWriter {
public final String node;
final Cell[] cells;
Map<String, Map<String, List<ReplicaStat>>> replicaInfo;
List<Clause> violations = new ArrayList<>();
boolean anyValueMissing = false;
Row(String node, List<String> params, NodeValueProvider snitch) {
replicaInfo = snitch.getReplicaCounts(node, params);
if (replicaInfo == null) replicaInfo = Collections.emptyMap();
this.node = node;
cells = new Cell[params.size()];
Map<String, Object> vals = new HashMap<>();
for (String param : params) vals.put(param, null);
snitch.getValues(node, vals);
Map<String, Object> vals = snitch.getValues(node, params);
for (int i = 0; i < params.size(); i++) {
String s = params.get(i);
cells[i] = new Cell(i, s, vals.get(s));
if (NODE.equals(s)) cells[i].val = node;
if (cells[i].val == null) anyValueMissing = true;
}
}
Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaStat>>> replicaInfo) {
this.node = node;
this.cells = new Cell[cells.length];
for (int i = 0; i < this.cells.length; i++) {
this.cells[i] = cells[i].copy();
}
this.anyValueMissing = anyValueMissing;
this.replicaInfo = replicaInfo;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put(node, (IteratorWriter) iw -> {
iw.add((MapWriter) e -> e.put("replicas", replicaInfo));
for (Cell cell : cells) iw.add(cell);
});
}
public Row copy() {
return new Row(node, cells, anyValueMissing, replicaInfo);
}
public Object getVal(String name) {
for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
return null;
}
}
static class Cell implements MapWriter {
@ -292,15 +252,60 @@ public class RuleSorter {
this.val = val;
}
Cell(int index, String name, Object val, Object val_) {
this.index = index;
this.name = name;
this.val = val;
this.val_ = val_;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put(name, val);
}
public Cell copy() {
return new Cell(index, name, val, val_);
}
}
static class Operation {
CollectionAction action;
String node, collection, shard, replica;
String targetNode;
}
static class ReplicaStat implements MapWriter {
final String name;
Map<String, Object> variables;
ReplicaStat(String name, Map<String, Object> vals) {
this.name = name;
this.variables = vals;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put(name, variables);
}
}
interface NodeValueProvider {
Map<String, Object> getValues(String node, Collection<String> keys);
void getValues(String node, Map<String, Object> valuesMap);
/**
* Get the details of each replica in a node. It attempts to fetch as much details about
* the replica as mentioned in the keys list
* <p>
* the format is {collection:shard :[{replicaetails}]}
*/
Map<String, Map<String, List<ReplicaStat>>> getReplicaCounts(String node, Collection<String> keys);
}
private static final Set<CollectionAction> supportedActions = new HashSet<>(Arrays.asList(ADDREPLICA, DELETEREPLICA, MOVEREPLICA, SPLITSHARD));
}

102
solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java
View File

@ -18,18 +18,23 @@
package org.apache.solr.recipe;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
public class TestRuleSorter extends SolrTestCaseJ4 {
public void testRuleParsing() {
public void testRuleParsing() throws IOException {
String rules = "{" +
"conditions:[{node:'!overseer', strict:false}, " +
"conditions:[{nodeRole:'!overseer', strict:false}, " +
"{replica:'<2',node:'*', shard:'#EACH'}]," +
" preferences:[" +
"{minimize:cores , precision:2}," +
@ -37,19 +42,93 @@ public class TestRuleSorter extends SolrTestCaseJ4 {
"{minimize:heap, precision:1000}]}";
Map nodeValues = (Map) Utils.fromJSONString( "{" +
Map<String,Map> nodeValues = (Map<String, Map>) Utils.fromJSONString( "{" +
"node1:{cores:12, freedisk: 334, heap:10480}," +
"node2:{cores:4, freedisk: 749, heap:6873}," +
"node3:{cores:7, freedisk: 262, heap:7834}," +
"node4:{cores:8, freedisk: 375, heap:16900}" +
"}");
String clusterState = "{'gettingstarted':{" +
" 'router':{'name':'compositeId'}," +
" 'shards':{" +
" 'shard1':{" +
" 'range':'80000000-ffffffff'," +
" 'state':'active'," +
" 'replicas':{" +
" 'core_node1':{" +
" 'core':'gettingstarted_shard1_replica1'," +
" 'base_url':'http://10.0.0.4:8983/solr'," +
" 'node_name':'node1'," +
" 'state':'active'," +
" 'leader':'true'}," +
" 'core_node4':{" +
" 'core':'gettingstarted_shard1_replica2'," +
" 'base_url':'http://10.0.0.4:7574/solr'," +
" 'node_name':'node2'," +
" 'state':'active'}}}," +
" 'shard2':{" +
" 'range':'0-7fffffff'," +
" 'state':'active'," +
" 'replicas':{" +
" 'core_node2':{" +
" 'core':'gettingstarted_shard2_replica1'," +
" 'base_url':'http://10.0.0.4:8983/solr'," +
" 'node_name':'node1'," +
" 'state':'active'," +
" 'leader':'true'}," +
" 'core_node3':{" +
" 'core':'gettingstarted_shard2_replica2'," +
" 'base_url':'http://10.0.0.4:7574/solr'," +
" 'node_name':'node2'," +
" 'state':'active'}}}}}}";
ValidatingJsonMap m = ValidatingJsonMap
.getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
RuleSorter ruleSorter = new RuleSorter((Map<String, Object>) Utils.fromJSONString(rules));
RuleSorter.Session session = ruleSorter.createSession(Arrays.asList("node1", "node2","node3","node4"), (node, valuesMap) -> {
Map n = (Map) nodeValues.get(node);
valuesMap.entrySet().stream().forEach(e -> e.setValue(n.get(e.getKey())));
});
session.sort();
RuleSorter.Session session;
RuleSorter.NodeValueProvider snitch = new RuleSorter.NodeValueProvider() {
@Override
public Map<String,Object> getValues(String node, Collection<String> keys) {
Map<String,Object> result = new LinkedHashMap<>();
keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
return result;
}
@Override
public Map<String, Map<String, List<RuleSorter.ReplicaStat>>> getReplicaCounts(String node, Collection<String> keys) {
Map<String, Map<String, List<RuleSorter.ReplicaStat>>> result = new LinkedHashMap<>();
m.forEach((collName, o) -> {
ValidatingJsonMap coll = (ValidatingJsonMap) o;
coll.getMap("shards").forEach((shard, o1) -> {
ValidatingJsonMap sh = (ValidatingJsonMap) o1;
sh.getMap("replicas").forEach((replicaName, o2) -> {
ValidatingJsonMap r = (ValidatingJsonMap) o2;
String node_name = (String) r.get("node_name");
if (!node_name.equals(node)) return;
Map<String, List<RuleSorter.ReplicaStat>> shardVsReplicaStats = result.get(collName);
if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
List<RuleSorter.ReplicaStat> replicaStats = shardVsReplicaStats.get(shard);
if (replicaStats == null) shardVsReplicaStats.put(shard, replicaStats = new ArrayList<>());
replicaStats.add(new RuleSorter.ReplicaStat(replicaName, new HashMap<>()));
});
});
});
return result;
}
};
session = ruleSorter.createSession(Arrays.asList("node1", "node2", "node3", "node4"), snitch);
session.applyRules();
List<RuleSorter.Row> l = session.getSorted();
assertEquals("node1",l.get(0).node);
assertEquals("node3",l.get(1).node);
@ -57,11 +136,10 @@ public class TestRuleSorter extends SolrTestCaseJ4 {
assertEquals("node2",l.get(3).node);
// System.out.println(session);
System.out.printf(Utils.toJSONString(Utils.getDeepCopy(session.toMap(new LinkedHashMap<>()), 8)));
System.out.println(Utils.getDeepCopy(session.getViolations(), 6));
}
}