Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed and added test for greedy conditions

This commit is contained in:
Noble Paul 2017-05-25 17:55:00 +09:30
parent fe217668cb
commit 9ad811b36f
5 changed files with 215 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
View File

@ -38,6 +38,8 @@ class AddReplicaSuggester extends Suggester {
throw new RuntimeException("add-replica requires 'collection' and 'shard'");
//iterate through elements and identify the least loaded
List<Clause.Violation> leastSeriousViolation = null;
Integer targetNodeIndex = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
@ -45,14 +47,21 @@ class AddReplicaSuggester extends Suggester {
tmpRow.violations.clear();
List<Clause.Violation> errs = testChangedRow(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
if (!containsNewErrors(errs)) {// there are no rule violations
getMatrix().set(i, getMatrix().get(i).addReplica(coll, shard));
return CollectionAdminRequest
.addReplicaToShard(coll, shard)
.setNode(row.node);
if(!containsNewErrors(errs)) {
if(isLessSerious(errs, leastSeriousViolation)){
leastSeriousViolation = errs;
targetNodeIndex = i;
}
}
}
if (targetNodeIndex != null) {// there are no rule violations
getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
return CollectionAdminRequest
.addReplicaToShard(coll, shard)
.setNode(getMatrix().get(targetNodeIndex).node);
}
return null;
}

47
solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
View File

@ -71,7 +71,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
collection = parse(COLLECTION, m);
shard = parse(SHARD, m);
this.replica = parse(REPLICA, m);
Condition replica = parse(REPLICA, m);
try {
int replicaCount = Integer.parseInt(String.valueOf(replica.val));
this.replica = new Condition(replica.name, replicaCount, replica.op);
} catch (NumberFormatException e) {
throw new RuntimeException("Only an integer value is supported for replica "+Utils.toJSONString(m));
}
m.forEach((s, o) -> parseCondition(s, o));
}
if (tag == null)
@ -102,12 +108,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
try {
int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
if (v != 0) return v;
return this.isPerCollectiontag() && that.isPerCollectiontag() ?
Integer.compare(this.replica.op.priority, that.replica.op.priority) :
0;
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
if(v ==0) {
v = Integer.compare((Integer)this.replica.val, (Integer)that.replica.val);
v = this.replica.op == LESS_THAN ? v : v * -1;
}
return v;
} else {
return 0;
}
} catch (NullPointerException e) {
System.out.println("this: " + Utils.toJSONString(this));
System.out.println("thAt: " + Utils.toJSONString(that));
throw e;
}
}
@ -147,6 +158,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
return false;
}
public Integer delta(Object val) {
return op.delta(this.val, val);
}
}
static Condition parse(String s, Map m) {
@ -179,13 +194,15 @@ public class Clause implements MapWriter, Comparable<Clause> {
public class Violation implements MapWriter {
final String shard, coll, node;
final Object actualVal;
final Integer delta;//how far is the actual value from the expected value
private final int hash;
private Violation(String coll, String shard, String node, Object actualVal) {
private Violation(String coll, String shard, String node, Object actualVal, Integer delta ) {
this.shard = shard;
this.coll = coll;
this.node = node;
this.delta = delta;
this.actualVal = actualVal;
hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
@ -215,8 +232,11 @@ public class Clause implements MapWriter, Comparable<Clause> {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
ew.putIfNotNull("violation", (MapWriter) ew1 -> ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
String.valueOf(actualVal)));
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
String.valueOf(actualVal));
ew1.putIfNotNull("delta", delta);
});
ew.put("clause", getClause());
}
}
@ -233,7 +253,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
if (!replica.isPass(counts.getValue())) {
errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
tag.name.equals("node") ? counts.getKey() : null, counts.getValue()));
tag.name.equals("node") ? counts.getKey() : null, counts.getValue(), replica.delta(counts.getValue())));
}
}
}
@ -241,7 +261,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
for (Row r : allRows) {
if (!tag.isPass(r)) {
errors.add(new Violation(null, null, r.node, r.getVal(tag.name) ));
errors.add(new Violation(null, null, r.node, r.getVal(tag.name) , tag.delta(r.getVal(tag.name))));
}
}
}
@ -264,11 +284,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (!shard.isPass(shardName)) break;
collMap.putIfAbsent(shardName, new HashMap<>());
Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
AtomicInteger count = null;
Object tagVal = row.getVal(tag.name);
tagVsCount.putIfAbsent(tag.isPass(tagVal)? String.valueOf(tagVal) : "", new AtomicInteger());
if (tag.isPass(tagVal)) {
tagVsCount.put(String.valueOf(tagVal), count = tagVsCount.getOrDefault(tagVal, new AtomicInteger()));
count.addAndGet(shards.getValue().size());
tagVsCount.get(tagVal).addAndGet(shards.getValue().size());
}
}
}

36
solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
View File

@ -40,12 +40,23 @@ public enum Operand {
return ANY.equals(val) || Policy.EACH.equals(val) ? val : null;
}
},
EQUAL("", 0),
EQUAL("", 0) {
@Override
public int _delta(int expected, int actual) {
return expected - actual;
}
},
NOT_EQUAL("!", 2) {
@Override
public TestStatus match(Object ruleVal, Object testVal) {
return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
}
@Override
public int _delta(int expected, int actual) {
return expected - actual;
}
},
GREATER_THAN(">", 1) {
@Override
@ -60,6 +71,10 @@ public enum Operand {
return compareNum(ruleVal, testVal) == 1 ? PASS : FAIL;
}
@Override
protected int _delta(int expected, int actual) {
return actual > expected ? 0 : (expected + 1) - actual;
}
},
LESS_THAN("<", 2) {
@Override
@ -68,6 +83,11 @@ public enum Operand {
return compareNum(ruleVal, testVal) == -1 ? PASS : FAIL;
}
@Override
protected int _delta(int expected, int actual) {
return actual < expected ? 0 : (expected ) - actual;
}
@Override
public Object parse(String val) {
return checkNumeric(super.parse(val));
@ -118,4 +138,18 @@ public enum Operand {
}
return o;
}
public Integer delta(Object expected, Object actual) {
try {
Integer expectedInt = Integer.parseInt(String.valueOf(expected));
Integer actualInt = Integer.parseInt(String.valueOf(actual));
return _delta(expectedInt, actualInt);
} catch (Exception e) {
return null;
}
}
protected int _delta(int expected, int actual) {
return 0;
}
}

39
solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
View File

@ -348,13 +348,20 @@ public class Policy implements MapWriter {
public SolrRequest getOperation() {
if (!isInitialized) {
String coll = (String) hints.get(Hint.COLL);
if(coll != null){
// if this is not a known collection from the existing clusterstate,
// then add it
if(session.matrix.stream().noneMatch(row -> row.replicaInfo.containsKey(coll))){
session.matrix.get(0).replicaInfo.put(coll, new HashMap<>());
session.addClausesForCollection(session.dataProvider, coll);
Collections.sort(session.expandedClauses);
String shard = (String) hints.get(Hint.SHARD);
// if this is not a known collection from the existing clusterstate,
// then add it
if(session.matrix.stream().noneMatch(row -> row.replicaInfo.containsKey(coll))){
session.addClausesForCollection(session.dataProvider, coll);
Collections.sort(session.expandedClauses);
}
if(coll != null) {
for (Row row : session.matrix) {
if (!row.replicaInfo.containsKey(coll)) row.replicaInfo.put(coll, new HashMap<>());
if(shard != null){
Map<String, List<ReplicaInfo>> shardInfo = row.replicaInfo.get(coll);
if(!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
}
}
}
session.applyRules();
@ -373,7 +380,23 @@ public class Policy implements MapWriter {
return session.matrix;
}
boolean containsNewErrors(List<Clause.Violation> errs){
boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
if (old == null || fresh.size() < old.size()) return true;
if(fresh.size() == old.size()){
for (int i = 0; i < old.size(); i++) {
if(fresh.get(i).equals(old.get(i))) {
if (fresh.get(i) != null &&
old.get(i).delta != null &&
Math.abs(fresh.get(i).delta) < Math.abs(old.get(i).delta))
return true;
}
}
}
return false;
}
boolean containsNewErrors(List<Violation> errs){
for (Clause.Violation err : errs) {
if(!originalViolations.contains(err)) return true;
}

145
solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
View File

@ -55,7 +55,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertFalse(c.replica.isPass(2));
assertFalse(c.replica.isPass(1));
c = new Clause((Map<String, Object>) Utils.fromJSONString("{nodeRole:'!overseer'}"));
c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
assertTrue(c.tag.isPass("OVERSEER"));
assertFalse(c.tag.isPass("overseer"));
}
@ -96,12 +96,12 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
assertEquals("#ANY", clauses.get(3).original.get("shard"));
assertEquals("rack1",clauses.get(2).original.get("rack"));
assertEquals("rack1", clauses.get(2).original.get("rack"));
assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
}
public void testConditionsSort(){
public void testConditionsSort() {
String rules = "{" +
" 'cluster-policy':[" +
" { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
@ -112,9 +112,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
Collections.sort(clauses);
assertEquals("nodeRole", clauses.get(0).tag.name);
assertEquals("rack", clauses.get(1).tag.name);
assertEquals("nodeRole", clauses.get(1).tag.name);
assertEquals("rack", clauses.get(0).tag.name);
}
public static String clusterState = "{'gettingstarted':{" +
" 'router':{'name':'compositeId'}," +
" 'shards':{" +
@ -188,8 +189,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = session.getViolations();
assertEquals(3, violations.size());
assertTrue( violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
assertTrue( violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.op == Operand.LESS_THAN && "node".equals(violation.getClause().tag.name))));
Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
@ -214,8 +215,64 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
public void testMoveReplica(){
public void testGreedyConditions() {
String autoscaleJson = "{" +
" 'cluster-policy':[" +
" {'cores':'<10','node':'#ANY'}," +
" {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
" { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
" {'nodeRole':'overseer','replica':'0'}]," +
" 'cluster-preferences':[" +
" {'minimize':'cores', 'precision':3}," +
" {'maximize':'freedisk','precision':100}]}";
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
"node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
"node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
"node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
"}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
ClusterDataProvider cdp = new ClusterDataProvider() {
@Override
public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
return clusterDataProvider.getNodeValues(node, tags);
}
@Override
public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return clusterDataProvider.getReplicaInfo(node, keys);
}
@Override
public Collection<String> getNodes() {
return clusterDataProvider.getNodes();
}
@Override
public String getPolicy(String coll) {
return null;
}
};
Policy.Session session = policy.createSession(cdp);
Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
SolrRequest op = suggester
.hint(Hint.COLL, "newColl")
.hint(Hint.SHARD, "shard1")
.getOperation();
assertNotNull(op);
assertEquals("node3", op.getParams().get("node"));
op = suggester
.hint(Hint.COLL, "newColl")
.hint(Hint.SHARD, "shard1")
.getOperation();
assertNotNull(op);
assertEquals("node3", op.getParams().get("node"));
}
public void testMoveReplica() {
String autoscaleJson = "{" +
" 'cluster-policy':[" +
" {'cores':'<10','node':'#ANY'}," +
@ -232,17 +289,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
" {'core_node2':{}}]}}}");
Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
m.put("shard1", Arrays.asList(
new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
));
new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
));
Map<String, Map<String,Object>> tagsMap = (Map) Utils.fromJSONString( "{" +
" '127.0.0.1:60099_solr':{" +
" 'cores':0," +
" 'freedisk':918005641216}," +
" '127.0.0.1:60089_solr':{" +
" 'cores':2," +
" 'freedisk':918005641216}}}");
Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
" '127.0.0.1:60099_solr':{" +
" 'cores':0," +
" 'freedisk':918005641216}," +
" '127.0.0.1:60089_solr':{" +
" 'cores':2," +
" 'freedisk':918005641216}}}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
Policy.Session session = policy.createSession(new ClusterDataProvider() {
@ -268,7 +325,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
});
Policy.Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
.hint(Policy.Suggester.Hint.TARGET_NODE, "127.0.0.1:60099_solr");
.hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
SolrParams op = suggester.getOperation().getParams();
assertNotNull(op);
session = suggester.getSession();
@ -277,7 +334,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(op);
}
public void testOtherTag(){
public void testOtherTag() {
String rules = "{" +
"'cluster-preferences':[" +
"{'minimize':'cores','precision':2}," +
@ -285,13 +342,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
"{'minimize':'heapUsage','precision':1000}" +
"]," +
"'cluster-policy':[" +
"{'nodeRole':'!overseer','strict':false}," +
"{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}" +
"]," +
"'policies':{" +
"'p1':[" +
"{'nodeRole':'!overseer','strict':false}," +
"{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
"{'replica':'<3','shard':'#EACH','rack':'#ANY'}" +
@ -328,7 +385,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
return "p1";
}
};
Policy.Session session = policy.createSession(cdp);
Policy.Session session = policy.createSession(cdp);
CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session
.getSuggester(ADDREPLICA)
@ -339,19 +396,19 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
return new ClusterDataProvider() {
@Override
public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
Map<String, Object> result = new LinkedHashMap<>();
tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
return result;
}
@Override
public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
Map<String, Object> result = new LinkedHashMap<>();
tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
return result;
}
@Override
public Collection<String> getNodes() {
return nodeValues.keySet();
}
@Override
public Collection<String> getNodes() {
return nodeValues.keySet();
}
@Override
public String getPolicy(String coll) {
@ -359,15 +416,15 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
}
public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
}
};
};
}
public void testMultiReplicaPlacement() {
String autoScaleJson ="{" +
String autoScaleJson = "{" +
" 'cluster-preferences': [" +
" { maximize : freedisk , precision: 50}," +
" { minimize : cores, precision: 2}" +
@ -387,7 +444,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
Map<String,Map> nodeValues = (Map<String, Map>) Utils.fromJSONString( "{" +
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heap:10480, rack:rack3}," +
"node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, rack:rack1}," +
"node3:{cores:7, freedisk: 262, heap:7834, rack:rack4}," +
@ -420,8 +477,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
"newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3);
assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2","node1","node3")));
assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2","node1","node3")));
assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2", "node1", "node3")));
assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2", "node1", "node3")));
}
@ -443,7 +500,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.get(shard);
if (replicaInfos == null) shardVsReplicaStats.put(shard, replicaInfos = new ArrayList<>());
replicaInfos.add(new Policy.ReplicaInfo(replicaName,collName, shard, new HashMap<>()));
replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, new HashMap<>()));
});
});
});