Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules

This commit is contained in:
Noble Paul 2018-07-16 17:29:25 +10:00
parent f291f3eb6c
commit 5b9f4f3ecf
12 changed files with 468 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -110,6 +110,8 @@ New Features
* SOLR-12441: New NestedUpdateProcessorFactory (URP) to populate special fields _nest_parent_ and _nest_path_ of nested
(child) documents. It will generate a uniqueKey of nested docs if they were blank too. (Moshe Bla, David Smiley)
* SOLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules (noble)
Bug Fixes
----------------------

9
solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
View File

@ -119,8 +119,15 @@ public class TestPolicyCloud extends SolrCloudTestCase {
Policy.Session session = config.getPolicy().createSession(cloudManager);
for (Row row : session.getSortedNodes()) {
Object val = row.getVal(Suggestion.ConditionType.TOTALDISK.tagName, null);
log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk",null));
assertTrue(val != null);
}
count .set(0);
for (Row row : session.getSorted()) {
for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();

2
solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
View File

@ -92,7 +92,7 @@ public class TestPolicyCloud extends SimSolrCloudTestCase {
Policy.Session session = config.getPolicy().createSession(cluster);
AtomicInteger count = new AtomicInteger(0);
for (Row row : session.getSorted()) {
for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();

101
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
View File

@ -30,6 +30,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.util.StrUtils;
@ -116,6 +117,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
public static Clause create(String json) {
return create((Map<String, Object>) Utils.fromJSONString(json));
}
public static Clause create(Map<String, Object> m) {
Clause clause = new Clause(m);
return clause.hasComputedValue() ?
@ -150,7 +155,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
private Condition evaluateValue(Condition condition, Function<Condition, Object> computedValueEvaluator) {
if (condition == null) return null;
if (condition.computationType == null) return condition;
if (condition.computedType == null) return condition;
Object val = computedValueEvaluator.apply(condition);
val = condition.op.readRuleValue(new Condition(condition.name, val, condition.op, null, null));
return new Condition(condition.name, val, condition.op, null, this);
@ -182,9 +187,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
private boolean hasComputedValue() {
if (replica != null && replica.computationType != null) return true;
if (tag != null && tag.computationType != null) return true;
if (globalTag != null && globalTag.computationType != null) return true;
if (replica != null && replica.computedType != null) return true;
if (tag != null && tag.computedType != null) return true;
if (globalTag != null && globalTag.computedType != null) return true;
return false;
}
@ -233,9 +238,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
Condition parse(String s, Map m) {
Object expectedVal = null;
ComputationType computationType = null;
ComputedType computedType = null;
Object val = m.get(s);
Suggestion.ConditionType varType = Suggestion.getTagType(s);
ConditionType varType = Suggestion.getTagType(s);
if (varType.isHidden) {
throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed in a policy rule : ''{1}'' ", varType.tagName, Utils.toJSONString(m)));
}
try {
String conditionName = s.trim();
Operand operand = null;
@ -244,32 +252,39 @@ public class Clause implements MapWriter, Comparable<Clause> {
expectedVal = Policy.ANY;
} else if (val instanceof String) {
String strVal = ((String) val).trim();
val = strVal;
if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
else operand = Operand.EQUAL;
strVal = strVal.substring(Operand.EQUAL == operand || WILDCARD == operand ? 0 : 1);
for (ComputationType t : ComputationType.values()) {
for (ComputedType t : ComputedType.values()) {
String changedVal = t.match(strVal);
if (changedVal != null) {
computationType = t;
computedType = t;
strVal = changedVal;
if (varType == null || !varType.supportComputed(computationType, this)) {
throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in condition : ''{2}'' ",
if (varType == null || !varType.supportedComputedTypes.contains(computedType)) {
throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in clause : ''{2}'' ",
t, conditionName, Utils.toJSONString(m)));
}
}
}
operand = varType == null ? operand : varType.getOperand(operand, strVal, computationType);
expectedVal = validate(s, new Condition(s, strVal, operand, computationType, null), true);
if (computedType == null && ((String) val).charAt(0) == '#' && !varType.wildCards.contains(val)) {
throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not an allowed value for ''{1}'' , in clause : ''{2}'' . Supported value is : {3}",
val, conditionName, Utils.toJSONString(m), varType.wildCards));
}
operand = varType == null ? operand : varType.getOperand(operand, strVal, computedType);
expectedVal = validate(s, new Condition(s, strVal, operand, computedType, null), true);
} else if (val instanceof Number) {
operand = Operand.EQUAL;
operand = varType.getOperand(operand, val, null);
expectedVal = validate(s, new Condition(s, val, operand, null, null), true);
}
return new Condition(conditionName, expectedVal, operand, computationType, this);
return new Condition(conditionName, expectedVal, operand, computedType, this);
} catch (IllegalArgumentException iae) {
throw iae;
@ -290,16 +305,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
computedValueEvaluator.shardName = shardVsCount.getKey();
if (!shard.isPass(computedValueEvaluator.shardName)) continue;
for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
if(tag.varType.isPerNodeValue) computedValueEvaluator.node = counts.getKey();
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
ReplicaCount replicas = counts.getValue();
if (!sealedClause.replica.isPass(replicas)) {
Violation violation = new Violation(sealedClause,
computedValueEvaluator.collName,
computedValueEvaluator.shardName,
tag.name.equals("node") ? counts.getKey() : null,
tag.varType.isPerNodeValue ? computedValueEvaluator.node : null,
counts.getValue(),
sealedClause.getReplica().delta(replicas),
counts.getKey());
tag.varType.isPerNodeValue? null: counts.getKey());
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
}
}
@ -309,7 +325,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Row r : session.matrix) {
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
if (!sealedClause.getGlobalTag().isPass(r)) {
Suggestion.ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null,
ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null,
new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name), sealedClause.globalTag.delta(r.getVal(globalTag.name)), null)));
}
}
@ -322,6 +338,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
ComputedValueEvaluator computedValueEvaluator) {
Map<String, Map<String, Map<String, ReplicaCount>>> collVsShardVsTagVsCount = new HashMap<>();
for (Row row : allRows) {
computedValueEvaluator.node = row.node;
for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
String collectionName = colls.getKey();
if (!collection.isPass(collectionName)) continue;
@ -335,10 +352,19 @@ public class Clause implements MapWriter, Comparable<Clause> {
computedValueEvaluator.collName = collectionName;
computedValueEvaluator.shardName = shardName;
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
boolean pass = sealedClause.getTag().isPass(tagVal);
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
if (pass) {
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
Condition t = sealedClause.getTag();
if(t.varType.isPerNodeValue){
boolean pass = t.getOperand().match(t.val, tagVal) == TestStatus.PASS;
tagVsCount.computeIfAbsent(row.node, s -> new ReplicaCount());
if(pass) {
tagVsCount.get(row.node).increment(shards.getValue());
}
} else {
boolean pass = sealedClause.getTag().isPass(tagVal);
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
if (pass) {
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
}
}
}
}
@ -346,7 +372,8 @@ public class Clause implements MapWriter, Comparable<Clause> {
return collVsShardVsTagVsCount;
}
enum ComputationType {
enum ComputedType {
NULL(),
EQUAL() {
@Override
public String wrap(String value) {
@ -387,6 +414,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
@Override
public Object compute(Object val, Condition c) {
if (val == null || Clause.parseDouble(c.name, val) == 0) return 0d;
return Clause.parseDouble(c.name, val) * Clause.parseDouble(c.name, c.val).doubleValue() / 100;
}
@Override
public String toString() {
return "%";
@ -402,29 +435,34 @@ public class Clause implements MapWriter, Comparable<Clause> {
public String wrap(String value) {
return value;
}
public Object compute(Object val, Condition c) {
return val;
}
}
public static class Condition implements MapWriter {
final String name;
final Object val;
final Suggestion.ConditionType varType;
final ComputationType computationType;
final ConditionType varType;
final ComputedType computedType;
final Operand op;
private Clause clause;
Condition(String name, Object val, Operand op, ComputationType computationType, Clause parent) {
Condition(String name, Object val, Operand op, ComputedType computedType, Clause parent) {
this.name = name;
this.val = val;
this.op = op;
varType = Suggestion.getTagType(name);
this.computationType = computationType;
this.computedType = computedType;
this.clause = parent;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
String value = op.wrap(val);
if (computationType != null) value = computationType.wrap(value);
if (computedType != null) value = computedType.wrap(value);
ew.put(name, value);
}
@ -438,12 +476,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
boolean isPass(Object inputVal) {
if (computationType != null) {
if (computedType != null) {
throw new IllegalStateException("This is supposed to be called only from a Condition with no computed value or a SealedCondition");
}
if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(getClause().type);
if (varType == Suggestion.ConditionType.LAZY) { // we don't know the type
if (varType == ConditionType.LAZY) { // we don't know the type
return op.match(parseString(val), parseString(inputVal)) == PASS;
} else {
return op.match(val, validate(name, inputVal, false)) == PASS;
@ -514,14 +552,15 @@ public class Clause implements MapWriter, Comparable<Clause> {
final Policy.Session session;
String collName = null;
String shardName = null;
String node = null;
public ComputedValueEvaluator(Policy.Session session) {
this.session = session;
}
@Override
public Object apply(Condition computedValue) {
return computedValue.varType.computeValue(session, computedValue, collName, shardName);
public Object apply(Condition computedCondition) {
return computedCondition.varType.computeValue(session, computedCondition, collName, shardName, node);
}
}
@ -534,7 +573,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
*/
public static Object validate(String name, Object val, boolean isRuleVal) {
if (val == null) return null;
Suggestion.ConditionType info = Suggestion.getTagType(name);
ConditionType info = Suggestion.getTagType(name);
if (info == null) throw new RuntimeException("Unknown type :" + name);
return info.validate(name, val, isRuleVal);
}

16
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
View File

@ -38,6 +38,7 @@ import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
@ -87,7 +88,7 @@ public class Policy implements MapWriter {
final Map<String, List<Clause>> policies;
final List<Clause> clusterPolicy;
final List<Preference> clusterPreferences;
final List<Pair<String, Suggestion.ConditionType>> params;
final List<Pair<String, ConditionType>> params;
final List<String> perReplicaAttributes;
public Policy() {
@ -119,6 +120,15 @@ public class Policy implements MapWriter {
})
.collect(collectingAndThen(toList(), Collections::unmodifiableList));
for (String newParam : new ArrayList<>(newParams)) {
ConditionType t = Suggestion.getTagType(newParam);
if(t != null && !t.associatedPerNodeValues.isEmpty()){
for (String s : t.associatedPerNodeValues) {
if(!newParams.contains(s)) newParams.add(s);
}
}
}
this.policies = Collections.unmodifiableMap(
policiesFromMap((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault(POLICIES, emptyMap()), newParams));
this.params = Collections.unmodifiableList(newParams.stream()
@ -129,7 +139,7 @@ public class Policy implements MapWriter {
private List<String> readPerReplicaAttrs() {
return this.params.stream()
.map(s -> Suggestion.tagVsPerReplicaVal.get(s.first()))
.map(s -> s.second().perReplicaValue)
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
@ -571,7 +581,7 @@ public class Policy implements MapWriter {
return Utils.toJSONString(toMap(new LinkedHashMap<>()));
}
public List<Row> getSorted() {
public List<Row> getSortedNodes() {
return Collections.unmodifiableList(matrix);
}

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
View File

@ -182,7 +182,7 @@ public class PolicyHelper {
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
Policy.Session session = policy.createSession(cloudManager);
List<Row> sorted = session.getSorted();
List<Row> sorted = session.getSortedNodes();
List<Violation> violations = session.getViolations();
List<Preference> clusterPreferences = policy.getClusterPreferences();

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
View File

@ -96,7 +96,7 @@ public class Row implements MapWriter {
return null;
}
Object getVal(String name, Object def) {
public Object getVal(String name, Object def) {
for (Cell cell : cells)
if (cell.name.equals(name)) {
return cell.val == null ? def : cell.val;

330
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
View File

@ -17,9 +17,12 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
@ -29,15 +32,16 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.V2RequestSupport;
import org.apache.solr.client.solrj.cloud.autoscaling.Clause.ComputedType;
import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableSet;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@ -46,10 +50,41 @@ public class Suggestion {
public static final String coreidxsize = "INDEX.sizeInGB";
static final Map<String, ConditionType> validatetypes = new HashMap<>();
private static final String NULL = "";
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
@interface Meta {
String name();
Class type();
String[] associatedPerNodeValue() default NULL;
String associatedPerReplicaValue() default NULL;
String[] enumVals() default NULL;
String[] wildCards() default NULL;
boolean isNodeSpecificVal() default false;
boolean isHidden() default false;
boolean isAdditive() default true;
double min() default -1d;
double max() default -1d;
String metricsKey() default NULL;
ComputedType[] computedValues() default ComputedType.NULL;
}
public static ConditionType getTagType(String name) {
ConditionType info = validatetypes.get(name);
if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.LAZY;
if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.STRING;
if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = ConditionType.LAZY;
return info;
}
@ -57,7 +92,7 @@ public class Suggestion {
private static Object getOperandAdjustedValue(Object val, Object original) {
if (original instanceof Clause.Condition) {
Clause.Condition condition = (Clause.Condition) original;
if (condition.computationType == null && isIntegerEquivalent(val)) {
if (condition.computedType == null && isIntegerEquivalent(val)) {
if (condition.op == Operand.LESS_THAN) {
//replica : '<3'
val = val instanceof Long ?
@ -119,26 +154,32 @@ public class Suggestion {
}
public static final Map<String, String> tagVsPerReplicaVal = Stream.of(ConditionType.values())
.filter(tag -> tag.perReplicaValue != null)
.collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));
/**
* Type details of each variable in policies
*/
public enum ConditionType {
COLL("collection", String.class, null, null, null),
SHARD("shard", String.class, null, null, null),
REPLICA("replica", Double.class, null, 0L, null) {
@Meta(name = "collection",
type = String.class)
COLL(),
@Meta(
name = "shard",
type = String.class,
wildCards = {Policy.EACH, Policy.ANY})
SHARD(),
@Meta(name = "replica",
type = Double.class,
min = 0, max = -1,
computedValues = {ComputedType.EQUAL, ComputedType.PERCENT})
REPLICA() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
}
@Override
public Operand getOperand(Operand expected, Object strVal, Clause.ComputationType computationType) {
// if (computationType == Clause.ComputationType.EQUAL) return expected;
public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) {
if (strVal instanceof String) {
String s = ((String) strVal).trim();
int hyphenIdx = s.indexOf('-');
@ -152,24 +193,18 @@ public class Suggestion {
}
if (expected == Operand.EQUAL && (computationType != null || !isIntegerEquivalent(strVal))) {
if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) {
return Operand.RANGE_EQUAL;
}
if (expected == Operand.NOT_EQUAL && (computationType != null || !isIntegerEquivalent(strVal)))
if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal)))
return Operand.RANGE_NOT_EQUAL;
return expected;
}
@Override
public boolean supportComputed(Clause.ComputationType computedType, Clause clause) {
if (computedType == Clause.ComputationType.PERCENT || computedType == Clause.ComputationType.EQUAL) return true;
return false;
}
@Override
public String postValidate(Clause.Condition condition) {
if (condition.computationType == Clause.ComputationType.EQUAL) {
if (condition.computedType == ComputedType.EQUAL) {
if (condition.getClause().tag != null &&
condition.getClause().tag.varType == NODE &&
condition.getClause().tag.op == Operand.WILDCARD) {
@ -182,27 +217,53 @@ public class Suggestion {
}
@Override
public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard) {
if (cv.computationType == Clause.ComputationType.EQUAL) {
public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard, String node) {
if (cv.computedType == ComputedType.EQUAL) {
int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
if (relevantReplicasCount == 0) return 0;
return (double) session.matrix.size() / (double) relevantReplicasCount;
} else if (cv.computationType == Clause.ComputationType.PERCENT) {
int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
if (relevantReplicasCount == 0) return 0;
return (double) relevantReplicasCount * Clause.parseDouble(cv.name, cv.val).doubleValue() / 100;
} else if (cv.computedType == ComputedType.PERCENT) {
return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv);
} else {
throw new IllegalArgumentException("Unsupported type " + cv.computationType);
throw new IllegalArgumentException("Unsupported type " + cv.computedType);
}
}
},
PORT(ImplicitSnitch.PORT, Long.class, null, 1L, 65535L),
IP_1("ip_1", Long.class, null, 0L, 255L),
IP_2("ip_2", Long.class, null, 0L, 255L),
IP_3("ip_3", Long.class, null, 0L, 255L),
IP_4("ip_4", Long.class, null, 0L, 255L),
FREEDISK(ImplicitSnitch.DISK, Double.class, null, 0d, Double.MAX_VALUE, coreidxsize, Boolean.TRUE,null) {
@Meta(name = ImplicitSnitch.PORT,
type = Long.class,
min = 1,
max = 65535)
PORT(),
@Meta(name = "ip_1",
type = Long.class,
min = 0,
max = 255)
IP_1(),
@Meta(name = "ip_2",
type = Long.class,
min = 0,
max = 255)
IP_2(),
@Meta(name = "ip_3",
type = Long.class,
min = 0,
max = 255)
IP_3(),
@Meta(name = "ip_4",
type = Long.class,
min = 0,
max = 255)
IP_4(),
@Meta(name = ImplicitSnitch.DISK,
type = Double.class,
min = 0,
isNodeSpecificVal = true,
associatedPerReplicaValue = coreidxsize,
associatedPerNodeValue = "totaldisk",
computedValues = ComputedType.PERCENT)
FREEDISK() {
@Override
public Object convertVal(Object val) {
Number value = (Number) super.validate(ImplicitSnitch.DISK, val, false);
@ -212,6 +273,18 @@ public class Suggestion {
return value;
}
@Override
public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
if (condition.computedType == ComputedType.PERCENT) {
Row r = session.getNode(node);
if (r == null) return 0d;
return ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition);
}
throw new IllegalArgumentException("Unsupported type " + condition.computedType);
}
@Override
public int compareViolation(Violation v1, Violation v2) {
//TODO use tolerance compare
@ -288,14 +361,38 @@ public class Suggestion {
cell.val = currFreeDisk + idxSize;
}
},
CORE_IDX(coreidxsize, Double.class, null, 0d, Double.MAX_VALUE,null, false,"INDEX.sizeInBytes" ) {
@Meta(name = "totaldisk",
type = Double.class,
isHidden = true)
TOTALDISK() {
@Override
public Object convertVal(Object val) {
return FREEDISK.convertVal(val);
}
},
NODE_ROLE(ImplicitSnitch.NODEROLE, String.class, Collections.singleton("overseer"), null, null),
CORES(ImplicitSnitch.CORES, Long.class, null, 0L, Long.MAX_VALUE) {
@Meta(name = coreidxsize,
type = Double.class,
isNodeSpecificVal = true,
isHidden = true,
min = 0,
metricsKey = "INDEX.sizeInBytes")
CORE_IDX() {
@Override
public Object convertVal(Object val) {
return FREEDISK.convertVal(val);
}
},
@Meta(name = ImplicitSnitch.NODEROLE,
type = String.class,
enumVals = "overseer")
NODE_ROLE(),
@Meta(name = ImplicitSnitch.CORES,
type = Long.class,
min = 0)
CORES() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
@ -335,12 +432,33 @@ public class Suggestion {
cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
}
},
SYSLOADAVG(ImplicitSnitch.SYSLOADAVG, Double.class, null, 0d, 100d),
HEAPUSAGE(ImplicitSnitch.HEAPUSAGE, Double.class, null, 0d, null),
NUMBER("NUMBER", Long.class, null, 0L, Long.MAX_VALUE),
STRING("STRING", String.class, null, null, null),
NODE("node", String.class, null, null, null) {
@Meta(name = ImplicitSnitch.SYSLOADAVG,
type = Double.class,
min = 0,
max = 100,
isNodeSpecificVal = true)
SYSLOADAVG(),
@Meta(name = ImplicitSnitch.HEAPUSAGE,
type = Double.class,
min = 0,
isNodeSpecificVal = true)
HEAPUSAGE(),
@Meta(name = "NUMBER",
type = Long.class,
min = 0)
NUMBER(),
@Meta(name = "STRING",
type = String.class,
wildCards = Policy.EACH)
STRING(),
@Meta(name = "node",
type = String.class,
isNodeSpecificVal = true,
wildCards = {Policy.ANY, Policy.EACH})
NODE() {
@Override
public void getSuggestions(SuggestionCtx ctx) {
if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
@ -356,14 +474,17 @@ public class Suggestion {
}
@Override
/*@Override
public void addViolatingReplicas(ViolationCtx ctx) {
for (Row r : ctx.allRows) {
if(r.node.equals(ctx.tagKey)) collectViolatingReplicas(ctx,r);
}
}
}*/
},
LAZY("LAZY", null, null, null, null) {
@Meta(name = "LAZY",
type = void.class)
LAZY() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return Clause.parseString(val);
@ -374,38 +495,73 @@ public class Suggestion {
perNodeSuggestions(ctx);
}
},
DISKTYPE(ImplicitSnitch.DISKTYPE, String.class,
unmodifiableSet(new HashSet(Arrays.asList("ssd", "rotational"))), null, null) {
@Meta(name = ImplicitSnitch.DISKTYPE,
type = String.class,
enumVals = {"ssd", "rotational"})
DISKTYPE() {
@Override
public void getSuggestions(SuggestionCtx ctx) {
perNodeSuggestions(ctx);
}
};
final Class type;
final Set<String> vals;
final Number min;
final Number max;
final Boolean additive;
public final String tagName;
public final Class type;
public Meta meta;
public final Set<String> vals;
public final Number min;
public final Number max;
public final Boolean additive;
public final boolean isHidden;
public final Set<String> wildCards;
public final String perReplicaValue;
public final Set<String> associatedPerNodeValues;
public final String metricsAttribute;
public final boolean isPerNodeValue;
public final Set<ComputedType> supportedComputedTypes;
ConditionType(String tagName, Class type, Set<String> vals, Number min, Number max) {
this(tagName, type, vals, min, max, null, Boolean.TRUE, null);
ConditionType() {
try {
meta = ConditionType.class.getField(name()).getAnnotation(Meta.class);
if (meta == null) {
throw new RuntimeException("Invalid type, should have a @Meta annotation " + name());
}
} catch (NoSuchFieldException e) {
//cannot happen
}
this.tagName = meta.name();
this.type = meta.type();
this.vals = readSet(meta.enumVals());
this.max = readNum(meta.max());
this.min = readNum(meta.min());
this.perReplicaValue = readStr(meta.associatedPerReplicaValue());
this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
this.additive = meta.isAdditive();
this.metricsAttribute = readStr(meta.metricsKey());
this.isPerNodeValue = meta.isNodeSpecificVal();
this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ?
emptySet() :
unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
this.isHidden = meta.isHidden();
this.wildCards = readSet(meta.wildCards());
}
ConditionType(String tagName, Class type, Set<String> vals, Number min, Number max, String perReplicaValue,
Boolean additive, String metricsAttribute) {
this.tagName = tagName;
this.type = type;
this.vals = vals;
this.min = min;
this.max = max;
this.perReplicaValue = perReplicaValue;
this.additive = additive;
this.metricsAttribute = metricsAttribute;
private String readStr(String s) {
return NULL.equals(s) ? null : s;
}
private Number readNum(double v) {
return v == -1 ? null :
(Number) validate(null, v, true);
}
Set<String> readSet(String[] vals) {
if (NULL.equals(vals[0])) return emptySet();
return unmodifiableSet(new HashSet<>(Arrays.asList(vals)));
}
public void getSuggestions(SuggestionCtx ctx) {
@ -414,11 +570,12 @@ public class Suggestion {
public void addViolatingReplicas(ViolationCtx ctx) {
for (Row row : ctx.allRows) {
if (ctx.clause.tag.varType.isPerNodeValue && !row.node.equals(ctx.tagKey)) continue;
collectViolatingReplicas(ctx, row);
}
}
public Operand getOperand(Operand expected, Object val, Clause.ComputationType computationType) {
public Operand getOperand(Operand expected, Object val, ComputedType computedType) {
return expected;
}
@ -461,7 +618,7 @@ public class Suggestion {
}
return num;
} else if (type == String.class) {
if (isRuleVal && vals != null && !vals.contains(val))
if (isRuleVal && !vals.isEmpty() && !vals.contains(val))
throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(vals, ','));
return val;
} else {
@ -485,24 +642,32 @@ public class Suggestion {
return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1;
}
public boolean supportComputed(Clause.ComputationType computedType, Clause clause) {
return false;
}
public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard) {
public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
return condition.val;
}
}
private static void collectViolatingReplicas(ViolationCtx ctx, Row row) {
row.forEachReplica(replica -> {
if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
if (!ctx.currentViolation.matchShard(replica.getShard())) return;
if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
return;
ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
});
if (ctx.clause.tag.varType.isPerNodeValue) {
row.forEachReplica(replica -> {
if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
.withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
}
});
} else {
row.forEachReplica(replica -> {
if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
if (!ctx.currentViolation.matchShard(replica.getShard())) return;
if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
return;
ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
});
}
}
private static int getRelevantReplicasCount(Policy.Session session, Clause.Condition cv, String collection, String shard) {
@ -557,7 +722,12 @@ public class Suggestion {
}
}
/*public static final Map<String, String> tagVsPerReplicaVal = Stream.of(ConditionType.values())
.filter(tag -> tag.perReplicaValue != null)
.collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));*/
static {
for (Suggestion.ConditionType t : Suggestion.ConditionType.values()) Suggestion.validatetypes.put(t.tagName, t);
}
}

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
View File

@ -30,7 +30,7 @@ import org.apache.solr.common.util.Utils;
public class Violation implements MapWriter {
final String shard, coll, node;
final Object actualVal;
final Double replicaCountDelta;//how far is the actual value from the expected value
final Double replicaCountDelta;//how many extra replicas
final Object tagKey;
private final int hash;
private final Clause clause;

33
solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
View File

@ -58,6 +58,8 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.TOTALDISK;
/**
*
@ -215,18 +217,15 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
}
if (requestedTags.contains(ImplicitSnitch.DISKTYPE)) {
metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", new Function<Object, Pair<String,Object>>() {
@Override
public Pair<String, Object> apply(Object o) {
if("true".equals(String.valueOf(o))){
return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
}
if("false".equals(String.valueOf(o))){
return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
}
return new Pair<>(ImplicitSnitch.DISKTYPE,null);
metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", (Function<Object, Pair<String, Object>>) o -> {
if("true".equals(String.valueOf(o))){
return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
}
if("false".equals(String.valueOf(o))){
return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
}
return new Pair<>(ImplicitSnitch.DISKTYPE,null);
});
}
if (!metricsKeyVsTag.isEmpty()) {
@ -239,6 +238,10 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
groups.add("solr.node");
prefixes.add("CONTAINER.fs.usableSpace");
}
if (requestedTags.contains(TOTALDISK.tagName)) {
groups.add("solr.node");
prefixes.add("CONTAINER.fs.totalSpace");
}
if (requestedTags.contains(CORES)) {
groups.add("solr.core");
prefixes.add("CORE.coreName");
@ -260,9 +263,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
try {
SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
Map m = rsp.nl.asMap(4);
if (requestedTags.contains(DISK)) {
if (requestedTags.contains(FREEDISK.tagName)) {
Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.usableSpace");
if (n != null) ctx.getTags().put(DISK, Suggestion.getTagType(DISK).convertVal(n));
if (n != null) ctx.getTags().put(FREEDISK.tagName, FREEDISK.convertVal(n));
}
if (requestedTags.contains(TOTALDISK.tagName)) {
Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.totalSpace");
if (n != null) ctx.getTags().put(TOTALDISK.tagName, TOTALDISK.convertVal(n));
}
if (requestedTags.contains(CORES)) {
int count = 0;

5
solr/solrj/src/java/org/apache/solr/common/MapWriter.java
View File

@ -107,6 +107,11 @@ public interface MapWriter extends MapSerializable {
return this;
}
default EntryWriter putStringIfNotNull(String k, Object v) throws IOException {
if(v != null) put(k,String.valueOf(v));
return this;
}
default EntryWriter put(String k, int v) throws IOException {
put(k, (Integer) v);

117
solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
View File

@ -212,30 +212,30 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(Operand.RANGE_EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.01", null));
assertEquals(Operand.RANGE_NOT_EQUAL, REPLICA.getOperand(Operand.NOT_EQUAL, "2.01", null));
Clause clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '1.23', node:'#ANY'}"));
Clause clause = Clause.create("{replica: '1.23', node:'#ANY'}");
assertTrue(clause.getReplica().isPass(2));
assertTrue(clause.getReplica().isPass(1));
assertFalse(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(3));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '<1.23', node:'#ANY'}"));
clause = Clause.create("{replica: '<1.23', node:'#ANY'}");
assertTrue(clause.getReplica().isPass(1));
assertTrue(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(2));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '!1.23', node:'#ANY'}"));
clause = Clause.create("{replica: '!1.23', node:'#ANY'}");
assertFalse(clause.getReplica().isPass(2));
assertFalse(clause.getReplica().isPass(1));
assertTrue(clause.getReplica().isPass(0));
assertTrue(clause.getReplica().isPass(3));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: 1.23, node:'#ANY'}"));
clause = Clause.create("{replica: 1.23, node:'#ANY'}");
assertTrue(clause.getReplica().isPass(2));
assertTrue(clause.getReplica().isPass(1));
assertFalse(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(3));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '33%', node:'#ANY'}"));
clause = Clause.create("{replica: '33%', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.getReplica().op);
clause = clause.getSealedClause(condition -> {
if (condition.name.equals("replica")) {
@ -245,7 +245,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
});
assertTrue( clause.getReplica().isPass(2));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '3 - 5', node:'#ANY'}"));
clause = Clause.create("{replica: '3 - 5', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.getReplica().getOperand());
RangeVal range = (RangeVal) clause.getReplica().getValue();
assertEquals(3.0 , range.min);
@ -261,18 +261,27 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(new Double(0.0), clause.replica.delta(4));
expectThrows(IllegalArgumentException.class,
() -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '-33%', node:'#ANY'}")));
() -> Clause.create("{replica: '-33%', node:'#ANY'}"));
expectThrows(IllegalArgumentException.class,
() -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: 'x%', node:'#ANY'}")));
() -> Clause.create("{replica: 'x%', node:'#ANY'}"));
expectThrows(IllegalArgumentException.class,
() -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '20%-33%', node:'#ANY'}")));
() -> Clause.create("{replica: '20%-33%', node:'#ANY'}"));
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}"));
clause = Clause.create("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.replica.op);
clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', node:'#ANY'}"));
clause = Clause.create("{replica: '#EQUAL', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.replica.op);
expectThrows(IllegalArgumentException.class,
() -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', node:'node_1'}")));
() -> Clause.create("{replica: '#EQUAL', node:'node_1'}"));
clause = Clause.create("{replica : 0, freedisk:'<20%'}");
assertEquals(clause.tag.computedType, Clause.ComputedType.PERCENT);
assertEquals(clause.tag.op, Operand.LESS_THAN);
expectThrows(IllegalArgumentException.class,
() -> Clause.create("{replica : 0, INDEX.sizeInGB:'>300'}"));
expectThrows(IllegalArgumentException.class,
() -> Clause.create("{replica:'<3', shard: '#ANV', node:'#ANY'}"));
expectThrows(IllegalArgumentException.class,
() -> Clause.create("{replica:'<3', shard: '#EACH', node:'#E4CH'}"));
}
@ -428,36 +437,36 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
public void testOperands() {
Clause c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:'<2', node:'#ANY'}"));
Clause c = Clause.create("{replica:'<2', node:'#ANY'}");
assertFalse(c.replica.isPass(3));
assertFalse(c.replica.isPass(2));
assertTrue(c.replica.isPass(1));
assertEquals("{\"replica\":\"<2.0\"}", c.replica.toString());
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:'>2', node:'#ANY'}"));
c = Clause.create("{replica:'>2', node:'#ANY'}");
assertTrue(c.replica.isPass(3));
assertFalse(c.replica.isPass(2));
assertFalse(c.replica.isPass(1));
assertEquals("{\"replica\":\">2.0\"}", c.replica.toString());
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
c = Clause.create("{replica:0, nodeRole:'!overseer'}");
assertTrue(c.tag.isPass("OVERSEER"));
assertFalse(c.tag.isPass("overseer"));
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}"));
c = Clause.create("{replica:0, sysLoadAvg:'<12.7'}");
assertTrue(c.tag.isPass("12.6"));
assertTrue(c.tag.isPass(12.6d));
assertFalse(c.tag.isPass("12.9"));
assertFalse(c.tag.isPass(12.9d));
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}"));
c = Clause.create("{replica:0, sysLoadAvg:'>12.7'}");
assertTrue(c.tag.isPass("12.8"));
assertTrue(c.tag.isPass(12.8d));
assertFalse(c.tag.isPass("12.6"));
assertFalse(c.tag.isPass(12.6d));
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, 'metrics:x:y:z':'>12.7'}"));
c = Clause.create("{replica:0, 'metrics:x:y:z':'>12.7'}");
assertTrue(c.tag.val instanceof String);
assertTrue(c.tag.isPass("12.8"));
assertTrue(c.tag.isPass(12.8d));
@ -465,7 +474,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertFalse(c.tag.isPass(12.6d));
try {
c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, 'ip_1':'<30%'}"));
c = Clause.create("{replica:0, 'ip_1':'<30%'}");
fail("Expected exception");
} catch (Exception e) {
assertTrue(e.getMessage().contains("'%' is not allowed for variable : 'ip_1'"));
@ -1099,7 +1108,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy.Session session;
session = policy.createSession(getSolrCloudManager(nodeValues, clusterState));
List<Row> l = session.getSorted();
List<Row> l = session.getSortedNodes();
assertEquals("node1", l.get(0).node);
assertEquals("node3", l.get(1).node);
assertEquals("node4", l.get(2).node);
@ -1387,7 +1396,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
"{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" +
"{'replica':'<3','shard':'#EACH','sysprop.rack':'#EACH'}" +
"]" +
"}" +
"}";
@ -1871,6 +1880,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
assertEquals(1, violations.size());
assertEquals(4, violations.get(0).getViolatingReplicas().size());
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals(500d, r.delta, 0.1);
}
List<Suggester.SuggestionInfo> l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(3, l.size());
Map m = l.get(0).toMap(new LinkedHashMap<>());
@ -2873,4 +2886,66 @@ public void testUtilizeNodeFailure2() throws Exception {
}
public void testFreediskPercentage(){
String dataproviderdata = "{" +
" 'liveNodes': [" +
" 'node1:8983'," +
" 'node2:8984'," +
" 'node3:8985'" +
" ]," +
" 'replicaInfo': {" +
" 'node1:8983': {" +
" 'c1': {" +
" 's1': [" +
" {'r1': {'type': 'NRT'}}," +
" {'r2': {'type': 'NRT'}}" +
" ]," +
" 's2': [" +
" {'r1': {'type': 'NRT'}}," +
" {'r2': {'type': 'NRT'}}" +
" ]" +
" }" +
" }" +
" }," +
" 'nodeValues': {" +
" 'node1:8983': {" +
" 'cores': 4," +
" 'freedisk': 230," +
" 'totaldisk': 800," +
" 'port': 8983" +
" }," +
" 'node2:8984': {" +
" 'cores': 0," +
" 'freedisk': 1000," +
" 'totaldisk': 1200," +
" 'port': 8984" +
" }," +
" 'node3:8985': {" +
" 'cores': 0," +
" 'freedisk': 1500," +
" 'totaldisk': 1700," +
" 'port': 8985" +
" }" +
" }" +
"}";
String autoScalingjson = "{" +
" 'cluster-preferences': [" +
" { 'maximize': 'freedisk', 'precision': 50}," +
" { 'minimize': 'cores', 'precision': 3}" +
" ]," +
" 'cluster-policy': [" +
" { 'replica': 0, freedisk : '<30%'}" +
" ]" +
"}";
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) Utils.fromJSONString(dataproviderdata))).getViolations();
assertEquals(1, violations.size());
assertEquals(4, violations.get(0).getViolatingReplicas().size());
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals(10.0d, r.delta.doubleValue(), 0.1);
}
}
}