From 5b9f4f3ecf279b0566ffbec3df7ab4b64a09e969 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Mon, 16 Jul 2018 17:29:25 +1000 Subject: [PATCH] OLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules --- solr/CHANGES.txt | 2 + .../cloud/autoscaling/TestPolicyCloud.java | 9 +- .../autoscaling/sim/TestPolicyCloud.java | 2 +- .../solrj/cloud/autoscaling/Clause.java | 101 ++++-- .../solrj/cloud/autoscaling/Policy.java | 16 +- .../solrj/cloud/autoscaling/PolicyHelper.java | 2 +- .../client/solrj/cloud/autoscaling/Row.java | 2 +- .../solrj/cloud/autoscaling/Suggestion.java | 330 +++++++++++++----- .../solrj/cloud/autoscaling/Violation.java | 2 +- .../impl/SolrClientNodeStateProvider.java | 33 +- .../org/apache/solr/common/MapWriter.java | 5 + .../solrj/cloud/autoscaling/TestPolicy.java | 117 +++++-- 12 files changed, 468 insertions(+), 153 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b7df7b6aa55..d8823db7cf2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -110,6 +110,8 @@ New Features * SOLR-12441: New NestedUpdateProcessorFactory (URP) to populate special fields _nest_parent_ and _nest_path_ of nested (child) documents. It will generate a uniqueKey of nested docs if they were blank too. (Moshe Bla, David Smiley) +* SOLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules (noble) + Bug Fixes ---------------------- diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java index 19324e73c88..a25a67a4f68 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java @@ -119,8 +119,15 @@ public class TestPolicyCloud extends SolrCloudTestCase { Policy.Session session = config.getPolicy().createSession(cloudManager); + for (Row row : session.getSortedNodes()) { + Object val = row.getVal(Suggestion.ConditionType.TOTALDISK.tagName, null); + log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk",null)); + assertTrue(val != null); + + } + count .set(0); - for (Row row : session.getSorted()) { + for (Row row : session.getSortedNodes()) { row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> { for (ReplicaInfo replicaInfo : replicaInfos) { if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java index b6368d6688a..fad637d87cd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java @@ -92,7 +92,7 @@ public class TestPolicyCloud extends SimSolrCloudTestCase { Policy.Session session = config.getPolicy().createSession(cluster); AtomicInteger count = new AtomicInteger(0); - for (Row row : session.getSorted()) { + for (Row row : session.getSortedNodes()) { row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> { for (ReplicaInfo replicaInfo : replicaInfos) { if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java index 60ebf9b028f..06dc7a0a873 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType; import org.apache.solr.common.MapWriter; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.util.StrUtils; @@ -116,6 +117,10 @@ public class Clause implements MapWriter, Comparable { } } + public static Clause create(String json) { + return create((Map) Utils.fromJSONString(json)); + } + public static Clause create(Map m) { Clause clause = new Clause(m); return clause.hasComputedValue() ? @@ -150,7 +155,7 @@ public class Clause implements MapWriter, Comparable { private Condition evaluateValue(Condition condition, Function computedValueEvaluator) { if (condition == null) return null; - if (condition.computationType == null) return condition; + if (condition.computedType == null) return condition; Object val = computedValueEvaluator.apply(condition); val = condition.op.readRuleValue(new Condition(condition.name, val, condition.op, null, null)); return new Condition(condition.name, val, condition.op, null, this); @@ -182,9 +187,9 @@ public class Clause implements MapWriter, Comparable { } private boolean hasComputedValue() { - if (replica != null && replica.computationType != null) return true; - if (tag != null && tag.computationType != null) return true; - if (globalTag != null && globalTag.computationType != null) return true; + if (replica != null && replica.computedType != null) return true; + if (tag != null && tag.computedType != null) return true; + if (globalTag != null && globalTag.computedType != null) return true; return false; } @@ -233,9 +238,12 @@ public class Clause implements MapWriter, Comparable { Condition parse(String s, Map m) { Object expectedVal = null; - ComputationType computationType = null; + ComputedType computedType = null; Object val = m.get(s); - Suggestion.ConditionType varType = Suggestion.getTagType(s); + ConditionType varType = Suggestion.getTagType(s); + if (varType.isHidden) { + throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed in a policy rule : ''{1}'' ", varType.tagName, Utils.toJSONString(m))); + } try { String conditionName = s.trim(); Operand operand = null; @@ -244,32 +252,39 @@ public class Clause implements MapWriter, Comparable { expectedVal = Policy.ANY; } else if (val instanceof String) { String strVal = ((String) val).trim(); + val = strVal; if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD; else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL; else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN; else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN; else operand = Operand.EQUAL; strVal = strVal.substring(Operand.EQUAL == operand || WILDCARD == operand ? 0 : 1); - for (ComputationType t : ComputationType.values()) { + for (ComputedType t : ComputedType.values()) { String changedVal = t.match(strVal); if (changedVal != null) { - computationType = t; + computedType = t; strVal = changedVal; - if (varType == null || !varType.supportComputed(computationType, this)) { - throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in condition : ''{2}'' ", + if (varType == null || !varType.supportedComputedTypes.contains(computedType)) { + throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in clause : ''{2}'' ", t, conditionName, Utils.toJSONString(m))); } } } - operand = varType == null ? operand : varType.getOperand(operand, strVal, computationType); - expectedVal = validate(s, new Condition(s, strVal, operand, computationType, null), true); + if (computedType == null && ((String) val).charAt(0) == '#' && !varType.wildCards.contains(val)) { + throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not an allowed value for ''{1}'' , in clause : ''{2}'' . Supported value is : {3}", + val, conditionName, Utils.toJSONString(m), varType.wildCards)); + + + } + operand = varType == null ? operand : varType.getOperand(operand, strVal, computedType); + expectedVal = validate(s, new Condition(s, strVal, operand, computedType, null), true); } else if (val instanceof Number) { operand = Operand.EQUAL; operand = varType.getOperand(operand, val, null); expectedVal = validate(s, new Condition(s, val, operand, null, null), true); } - return new Condition(conditionName, expectedVal, operand, computationType, this); + return new Condition(conditionName, expectedVal, operand, computedType, this); } catch (IllegalArgumentException iae) { throw iae; @@ -290,16 +305,17 @@ public class Clause implements MapWriter, Comparable { computedValueEvaluator.shardName = shardVsCount.getKey(); if (!shard.isPass(computedValueEvaluator.shardName)) continue; for (Map.Entry counts : shardVsCount.getValue().entrySet()) { + if(tag.varType.isPerNodeValue) computedValueEvaluator.node = counts.getKey(); SealedClause sealedClause = getSealedClause(computedValueEvaluator); ReplicaCount replicas = counts.getValue(); if (!sealedClause.replica.isPass(replicas)) { Violation violation = new Violation(sealedClause, computedValueEvaluator.collName, computedValueEvaluator.shardName, - tag.name.equals("node") ? counts.getKey() : null, + tag.varType.isPerNodeValue ? computedValueEvaluator.node : null, counts.getValue(), sealedClause.getReplica().delta(replicas), - counts.getKey()); + tag.varType.isPerNodeValue? null: counts.getKey()); tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation)); } } @@ -309,7 +325,7 @@ public class Clause implements MapWriter, Comparable { for (Row r : session.matrix) { SealedClause sealedClause = getSealedClause(computedValueEvaluator); if (!sealedClause.getGlobalTag().isPass(r)) { - Suggestion.ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null, + ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null, new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name), sealedClause.globalTag.delta(r.getVal(globalTag.name)), null))); } } @@ -322,6 +338,7 @@ public class Clause implements MapWriter, Comparable { ComputedValueEvaluator computedValueEvaluator) { Map>> collVsShardVsTagVsCount = new HashMap<>(); for (Row row : allRows) { + computedValueEvaluator.node = row.node; for (Map.Entry>> colls : row.collectionVsShardVsReplicas.entrySet()) { String collectionName = colls.getKey(); if (!collection.isPass(collectionName)) continue; @@ -335,10 +352,19 @@ public class Clause implements MapWriter, Comparable { computedValueEvaluator.collName = collectionName; computedValueEvaluator.shardName = shardName; SealedClause sealedClause = getSealedClause(computedValueEvaluator); - boolean pass = sealedClause.getTag().isPass(tagVal); - tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount()); - if (pass) { - tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue()); + Condition t = sealedClause.getTag(); + if(t.varType.isPerNodeValue){ + boolean pass = t.getOperand().match(t.val, tagVal) == TestStatus.PASS; + tagVsCount.computeIfAbsent(row.node, s -> new ReplicaCount()); + if(pass) { + tagVsCount.get(row.node).increment(shards.getValue()); + } + } else { + boolean pass = sealedClause.getTag().isPass(tagVal); + tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount()); + if (pass) { + tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue()); + } } } } @@ -346,7 +372,8 @@ public class Clause implements MapWriter, Comparable { return collVsShardVsTagVsCount; } - enum ComputationType { + enum ComputedType { + NULL(), EQUAL() { @Override public String wrap(String value) { @@ -387,6 +414,12 @@ public class Clause implements MapWriter, Comparable { } } + @Override + public Object compute(Object val, Condition c) { + if (val == null || Clause.parseDouble(c.name, val) == 0) return 0d; + return Clause.parseDouble(c.name, val) * Clause.parseDouble(c.name, c.val).doubleValue() / 100; + } + @Override public String toString() { return "%"; @@ -402,29 +435,34 @@ public class Clause implements MapWriter, Comparable { public String wrap(String value) { return value; } + + public Object compute(Object val, Condition c) { + return val; + } + } public static class Condition implements MapWriter { final String name; final Object val; - final Suggestion.ConditionType varType; - final ComputationType computationType; + final ConditionType varType; + final ComputedType computedType; final Operand op; private Clause clause; - Condition(String name, Object val, Operand op, ComputationType computationType, Clause parent) { + Condition(String name, Object val, Operand op, ComputedType computedType, Clause parent) { this.name = name; this.val = val; this.op = op; varType = Suggestion.getTagType(name); - this.computationType = computationType; + this.computedType = computedType; this.clause = parent; } @Override public void writeMap(EntryWriter ew) throws IOException { String value = op.wrap(val); - if (computationType != null) value = computationType.wrap(value); + if (computedType != null) value = computedType.wrap(value); ew.put(name, value); } @@ -438,12 +476,12 @@ public class Clause implements MapWriter, Comparable { } boolean isPass(Object inputVal) { - if (computationType != null) { + if (computedType != null) { throw new IllegalStateException("This is supposed to be called only from a Condition with no computed value or a SealedCondition"); } if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(getClause().type); - if (varType == Suggestion.ConditionType.LAZY) { // we don't know the type + if (varType == ConditionType.LAZY) { // we don't know the type return op.match(parseString(val), parseString(inputVal)) == PASS; } else { return op.match(val, validate(name, inputVal, false)) == PASS; @@ -514,14 +552,15 @@ public class Clause implements MapWriter, Comparable { final Policy.Session session; String collName = null; String shardName = null; + String node = null; public ComputedValueEvaluator(Policy.Session session) { this.session = session; } @Override - public Object apply(Condition computedValue) { - return computedValue.varType.computeValue(session, computedValue, collName, shardName); + public Object apply(Condition computedCondition) { + return computedCondition.varType.computeValue(session, computedCondition, collName, shardName, node); } } @@ -534,7 +573,7 @@ public class Clause implements MapWriter, Comparable { */ public static Object validate(String name, Object val, boolean isRuleVal) { if (val == null) return null; - Suggestion.ConditionType info = Suggestion.getTagType(name); + ConditionType info = Suggestion.getTagType(name); if (info == null) throw new RuntimeException("Unknown type :" + name); return info.validate(name, val, isRuleVal); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java index f7b8c034d08..9addc67ef2b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import org.apache.solr.client.solrj.cloud.NodeStateProvider; import org.apache.solr.client.solrj.cloud.SolrCloudManager; +import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType; import org.apache.solr.client.solrj.impl.ClusterStateProvider; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.MapWriter; @@ -87,7 +88,7 @@ public class Policy implements MapWriter { final Map> policies; final List clusterPolicy; final List clusterPreferences; - final List> params; + final List> params; final List perReplicaAttributes; public Policy() { @@ -119,6 +120,15 @@ public class Policy implements MapWriter { }) .collect(collectingAndThen(toList(), Collections::unmodifiableList)); + for (String newParam : new ArrayList<>(newParams)) { + ConditionType t = Suggestion.getTagType(newParam); + if(t != null && !t.associatedPerNodeValues.isEmpty()){ + for (String s : t.associatedPerNodeValues) { + if(!newParams.contains(s)) newParams.add(s); + } + } + } + this.policies = Collections.unmodifiableMap( policiesFromMap((Map>>) jsonMap.getOrDefault(POLICIES, emptyMap()), newParams)); this.params = Collections.unmodifiableList(newParams.stream() @@ -129,7 +139,7 @@ public class Policy implements MapWriter { private List readPerReplicaAttrs() { return this.params.stream() - .map(s -> Suggestion.tagVsPerReplicaVal.get(s.first())) + .map(s -> s.second().perReplicaValue) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -571,7 +581,7 @@ public class Policy implements MapWriter { return Utils.toJSONString(toMap(new LinkedHashMap<>())); } - public List getSorted() { + public List getSortedNodes() { return Collections.unmodifiableList(matrix); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java index 7785fda5b4b..a965bf836bf 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java @@ -182,7 +182,7 @@ public class PolicyHelper { public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) { Policy.Session session = policy.createSession(cloudManager); - List sorted = session.getSorted(); + List sorted = session.getSortedNodes(); List violations = session.getViolations(); List clusterPreferences = policy.getClusterPreferences(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java index f1799c4199f..1adfdbf90b2 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java @@ -96,7 +96,7 @@ public class Row implements MapWriter { return null; } - Object getVal(String name, Object def) { + public Object getVal(String name, Object def) { for (Cell cell : cells) if (cell.name.equals(name)) { return cell.val == null ? def : cell.val; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java index f5e25cc2d80..97b26fca54f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java @@ -17,9 +17,12 @@ package org.apache.solr.client.solrj.cloud.autoscaling; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -29,15 +32,16 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.V2RequestSupport; +import org.apache.solr.client.solrj.cloud.autoscaling.Clause.ComputedType; import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr; import org.apache.solr.common.cloud.rule.ImplicitSnitch; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.StrUtils; +import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY; import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; @@ -46,10 +50,41 @@ public class Suggestion { public static final String coreidxsize = "INDEX.sizeInGB"; static final Map validatetypes = new HashMap<>(); + private static final String NULL = ""; + + @Target(ElementType.FIELD) + @Retention(RetentionPolicy.RUNTIME) + @interface Meta { + String name(); + + Class type(); + + String[] associatedPerNodeValue() default NULL; + + String associatedPerReplicaValue() default NULL; + + String[] enumVals() default NULL; + + String[] wildCards() default NULL; + + boolean isNodeSpecificVal() default false; + + boolean isHidden() default false; + + boolean isAdditive() default true; + + double min() default -1d; + + double max() default -1d; + + String metricsKey() default NULL; + + ComputedType[] computedValues() default ComputedType.NULL; + } public static ConditionType getTagType(String name) { ConditionType info = validatetypes.get(name); - if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.LAZY; + if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.STRING; if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = ConditionType.LAZY; return info; } @@ -57,7 +92,7 @@ public class Suggestion { private static Object getOperandAdjustedValue(Object val, Object original) { if (original instanceof Clause.Condition) { Clause.Condition condition = (Clause.Condition) original; - if (condition.computationType == null && isIntegerEquivalent(val)) { + if (condition.computedType == null && isIntegerEquivalent(val)) { if (condition.op == Operand.LESS_THAN) { //replica : '<3' val = val instanceof Long ? @@ -119,26 +154,32 @@ public class Suggestion { } - public static final Map tagVsPerReplicaVal = Stream.of(ConditionType.values()) - .filter(tag -> tag.perReplicaValue != null) - .collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue)); - /** * Type details of each variable in policies */ public enum ConditionType { - COLL("collection", String.class, null, null, null), - SHARD("shard", String.class, null, null, null), - REPLICA("replica", Double.class, null, 0L, null) { + @Meta(name = "collection", + type = String.class) + COLL(), + @Meta( + name = "shard", + type = String.class, + wildCards = {Policy.EACH, Policy.ANY}) + SHARD(), + + @Meta(name = "replica", + type = Double.class, + min = 0, max = -1, + computedValues = {ComputedType.EQUAL, ComputedType.PERCENT}) + REPLICA() { @Override public Object validate(String name, Object val, boolean isRuleVal) { return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val); } @Override - public Operand getOperand(Operand expected, Object strVal, Clause.ComputationType computationType) { -// if (computationType == Clause.ComputationType.EQUAL) return expected; + public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) { if (strVal instanceof String) { String s = ((String) strVal).trim(); int hyphenIdx = s.indexOf('-'); @@ -152,24 +193,18 @@ public class Suggestion { } - if (expected == Operand.EQUAL && (computationType != null || !isIntegerEquivalent(strVal))) { + if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) { return Operand.RANGE_EQUAL; } - if (expected == Operand.NOT_EQUAL && (computationType != null || !isIntegerEquivalent(strVal))) + if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) return Operand.RANGE_NOT_EQUAL; return expected; } - @Override - public boolean supportComputed(Clause.ComputationType computedType, Clause clause) { - if (computedType == Clause.ComputationType.PERCENT || computedType == Clause.ComputationType.EQUAL) return true; - return false; - } - @Override public String postValidate(Clause.Condition condition) { - if (condition.computationType == Clause.ComputationType.EQUAL) { + if (condition.computedType == ComputedType.EQUAL) { if (condition.getClause().tag != null && condition.getClause().tag.varType == NODE && condition.getClause().tag.op == Operand.WILDCARD) { @@ -182,27 +217,53 @@ public class Suggestion { } @Override - public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard) { - if (cv.computationType == Clause.ComputationType.EQUAL) { + public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard, String node) { + if (cv.computedType == ComputedType.EQUAL) { int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard); if (relevantReplicasCount == 0) return 0; return (double) session.matrix.size() / (double) relevantReplicasCount; - } else if (cv.computationType == Clause.ComputationType.PERCENT) { - int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard); - if (relevantReplicasCount == 0) return 0; - return (double) relevantReplicasCount * Clause.parseDouble(cv.name, cv.val).doubleValue() / 100; + } else if (cv.computedType == ComputedType.PERCENT) { + return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv); } else { - throw new IllegalArgumentException("Unsupported type " + cv.computationType); + throw new IllegalArgumentException("Unsupported type " + cv.computedType); } } }, - PORT(ImplicitSnitch.PORT, Long.class, null, 1L, 65535L), - IP_1("ip_1", Long.class, null, 0L, 255L), - IP_2("ip_2", Long.class, null, 0L, 255L), - IP_3("ip_3", Long.class, null, 0L, 255L), - IP_4("ip_4", Long.class, null, 0L, 255L), - FREEDISK(ImplicitSnitch.DISK, Double.class, null, 0d, Double.MAX_VALUE, coreidxsize, Boolean.TRUE,null) { + @Meta(name = ImplicitSnitch.PORT, + type = Long.class, + min = 1, + max = 65535) + PORT(), + @Meta(name = "ip_1", + type = Long.class, + min = 0, + max = 255) + IP_1(), + @Meta(name = "ip_2", + type = Long.class, + min = 0, + max = 255) + IP_2(), + @Meta(name = "ip_3", + type = Long.class, + min = 0, + max = 255) + IP_3(), + @Meta(name = "ip_4", + type = Long.class, + min = 0, + max = 255) + IP_4(), + + @Meta(name = ImplicitSnitch.DISK, + type = Double.class, + min = 0, + isNodeSpecificVal = true, + associatedPerReplicaValue = coreidxsize, + associatedPerNodeValue = "totaldisk", + computedValues = ComputedType.PERCENT) + FREEDISK() { @Override public Object convertVal(Object val) { Number value = (Number) super.validate(ImplicitSnitch.DISK, val, false); @@ -212,6 +273,18 @@ public class Suggestion { return value; } + @Override + public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) { + if (condition.computedType == ComputedType.PERCENT) { + Row r = session.getNode(node); + if (r == null) return 0d; + return ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition); + } + throw new IllegalArgumentException("Unsupported type " + condition.computedType); + } + + + @Override public int compareViolation(Violation v1, Violation v2) { //TODO use tolerance compare @@ -288,14 +361,38 @@ public class Suggestion { cell.val = currFreeDisk + idxSize; } }, - CORE_IDX(coreidxsize, Double.class, null, 0d, Double.MAX_VALUE,null, false,"INDEX.sizeInBytes" ) { + + @Meta(name = "totaldisk", + type = Double.class, + isHidden = true) + TOTALDISK() { @Override public Object convertVal(Object val) { return FREEDISK.convertVal(val); } }, - NODE_ROLE(ImplicitSnitch.NODEROLE, String.class, Collections.singleton("overseer"), null, null), - CORES(ImplicitSnitch.CORES, Long.class, null, 0L, Long.MAX_VALUE) { + + @Meta(name = coreidxsize, + type = Double.class, + isNodeSpecificVal = true, + isHidden = true, + min = 0, + metricsKey = "INDEX.sizeInBytes") + CORE_IDX() { + @Override + public Object convertVal(Object val) { + return FREEDISK.convertVal(val); + } + }, + @Meta(name = ImplicitSnitch.NODEROLE, + type = String.class, + enumVals = "overseer") + NODE_ROLE(), + + @Meta(name = ImplicitSnitch.CORES, + type = Long.class, + min = 0) + CORES() { @Override public Object validate(String name, Object val, boolean isRuleVal) { return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val); @@ -335,12 +432,33 @@ public class Suggestion { cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1; } }, - SYSLOADAVG(ImplicitSnitch.SYSLOADAVG, Double.class, null, 0d, 100d), - HEAPUSAGE(ImplicitSnitch.HEAPUSAGE, Double.class, null, 0d, null), - NUMBER("NUMBER", Long.class, null, 0L, Long.MAX_VALUE), - STRING("STRING", String.class, null, null, null), - NODE("node", String.class, null, null, null) { + @Meta(name = ImplicitSnitch.SYSLOADAVG, + type = Double.class, + min = 0, + max = 100, + isNodeSpecificVal = true) + SYSLOADAVG(), + + @Meta(name = ImplicitSnitch.HEAPUSAGE, + type = Double.class, + min = 0, + isNodeSpecificVal = true) + HEAPUSAGE(), + @Meta(name = "NUMBER", + type = Long.class, + min = 0) + NUMBER(), + @Meta(name = "STRING", + type = String.class, + wildCards = Policy.EACH) + STRING(), + + @Meta(name = "node", + type = String.class, + isNodeSpecificVal = true, + wildCards = {Policy.ANY, Policy.EACH}) + NODE() { @Override public void getSuggestions(SuggestionCtx ctx) { if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return; @@ -356,14 +474,17 @@ public class Suggestion { } - @Override + /*@Override public void addViolatingReplicas(ViolationCtx ctx) { for (Row r : ctx.allRows) { if(r.node.equals(ctx.tagKey)) collectViolatingReplicas(ctx,r); } - } + }*/ }, - LAZY("LAZY", null, null, null, null) { + + @Meta(name = "LAZY", + type = void.class) + LAZY() { @Override public Object validate(String name, Object val, boolean isRuleVal) { return Clause.parseString(val); @@ -374,38 +495,73 @@ public class Suggestion { perNodeSuggestions(ctx); } }, - DISKTYPE(ImplicitSnitch.DISKTYPE, String.class, - unmodifiableSet(new HashSet(Arrays.asList("ssd", "rotational"))), null, null) { + + @Meta(name = ImplicitSnitch.DISKTYPE, + type = String.class, + enumVals = {"ssd", "rotational"}) + DISKTYPE() { @Override public void getSuggestions(SuggestionCtx ctx) { perNodeSuggestions(ctx); } }; - final Class type; - final Set vals; - final Number min; - final Number max; - final Boolean additive; public final String tagName; + public final Class type; + public Meta meta; + + public final Set vals; + public final Number min; + public final Number max; + public final Boolean additive; + public final boolean isHidden; + public final Set wildCards; public final String perReplicaValue; + public final Set associatedPerNodeValues; public final String metricsAttribute; + public final boolean isPerNodeValue; + public final Set supportedComputedTypes; - ConditionType(String tagName, Class type, Set vals, Number min, Number max) { - this(tagName, type, vals, min, max, null, Boolean.TRUE, null); + ConditionType() { + try { + meta = ConditionType.class.getField(name()).getAnnotation(Meta.class); + if (meta == null) { + throw new RuntimeException("Invalid type, should have a @Meta annotation " + name()); + } + } catch (NoSuchFieldException e) { + //cannot happen + } + this.tagName = meta.name(); + this.type = meta.type(); + + this.vals = readSet(meta.enumVals()); + this.max = readNum(meta.max()); + this.min = readNum(meta.min()); + this.perReplicaValue = readStr(meta.associatedPerReplicaValue()); + this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue()); + this.additive = meta.isAdditive(); + this.metricsAttribute = readStr(meta.metricsKey()); + this.isPerNodeValue = meta.isNodeSpecificVal(); + this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ? + emptySet() : + unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues()))); + this.isHidden = meta.isHidden(); + this.wildCards = readSet(meta.wildCards()); } - ConditionType(String tagName, Class type, Set vals, Number min, Number max, String perReplicaValue, - Boolean additive, String metricsAttribute) { - this.tagName = tagName; - this.type = type; - this.vals = vals; - this.min = min; - this.max = max; - this.perReplicaValue = perReplicaValue; - this.additive = additive; - this.metricsAttribute = metricsAttribute; + private String readStr(String s) { + return NULL.equals(s) ? null : s; + } + + private Number readNum(double v) { + return v == -1 ? null : + (Number) validate(null, v, true); + } + + Set readSet(String[] vals) { + if (NULL.equals(vals[0])) return emptySet(); + return unmodifiableSet(new HashSet<>(Arrays.asList(vals))); } public void getSuggestions(SuggestionCtx ctx) { @@ -414,11 +570,12 @@ public class Suggestion { public void addViolatingReplicas(ViolationCtx ctx) { for (Row row : ctx.allRows) { + if (ctx.clause.tag.varType.isPerNodeValue && !row.node.equals(ctx.tagKey)) continue; collectViolatingReplicas(ctx, row); } } - public Operand getOperand(Operand expected, Object val, Clause.ComputationType computationType) { + public Operand getOperand(Operand expected, Object val, ComputedType computedType) { return expected; } @@ -461,7 +618,7 @@ public class Suggestion { } return num; } else if (type == String.class) { - if (isRuleVal && vals != null && !vals.contains(val)) + if (isRuleVal && !vals.isEmpty() && !vals.contains(val)) throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(vals, ',')); return val; } else { @@ -485,24 +642,32 @@ public class Suggestion { return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1; } - public boolean supportComputed(Clause.ComputationType computedType, Clause clause) { - return false; - } - - public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard) { + public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) { return condition.val; } } private static void collectViolatingReplicas(ViolationCtx ctx, Row row) { - row.forEachReplica(replica -> { - if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return; - if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return; - if (!ctx.currentViolation.matchShard(replica.getShard())) return; - if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard)) - return; - ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name)))); - }); + if (ctx.clause.tag.varType.isPerNodeValue) { + row.forEachReplica(replica -> { + if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) { + ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica) + .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name)))); + } + }); + } else { + row.forEachReplica(replica -> { + if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return; + if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return; + if (!ctx.currentViolation.matchShard(replica.getShard())) return; + if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard)) + return; + ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name)))); + }); + + } + + } private static int getRelevantReplicasCount(Policy.Session session, Clause.Condition cv, String collection, String shard) { @@ -557,7 +722,12 @@ public class Suggestion { } } + /*public static final Map tagVsPerReplicaVal = Stream.of(ConditionType.values()) + .filter(tag -> tag.perReplicaValue != null) + .collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));*/ static { for (Suggestion.ConditionType t : Suggestion.ConditionType.values()) Suggestion.validatetypes.put(t.tagName, t); } + + } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java index 87b0abe7fed..d4ab0e6e321 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java @@ -30,7 +30,7 @@ import org.apache.solr.common.util.Utils; public class Violation implements MapWriter { final String shard, coll, node; final Object actualVal; - final Double replicaCountDelta;//how far is the actual value from the expected value + final Double replicaCountDelta;//how many extra replicas final Object tagKey; private final int hash; private final Clause clause; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java index 903a75cc138..92183b86926 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java @@ -58,6 +58,8 @@ import org.slf4j.LoggerFactory; import static java.util.Collections.emptyMap; import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX; +import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK; +import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.TOTALDISK; /** * @@ -215,18 +217,15 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter } } if (requestedTags.contains(ImplicitSnitch.DISKTYPE)) { - metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", new Function>() { - @Override - public Pair apply(Object o) { - if("true".equals(String.valueOf(o))){ - return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational"); - } - if("false".equals(String.valueOf(o))){ - return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd"); - } - return new Pair<>(ImplicitSnitch.DISKTYPE,null); - + metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", (Function>) o -> { + if("true".equals(String.valueOf(o))){ + return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational"); } + if("false".equals(String.valueOf(o))){ + return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd"); + } + return new Pair<>(ImplicitSnitch.DISKTYPE,null); + }); } if (!metricsKeyVsTag.isEmpty()) { @@ -239,6 +238,10 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter groups.add("solr.node"); prefixes.add("CONTAINER.fs.usableSpace"); } + if (requestedTags.contains(TOTALDISK.tagName)) { + groups.add("solr.node"); + prefixes.add("CONTAINER.fs.totalSpace"); + } if (requestedTags.contains(CORES)) { groups.add("solr.core"); prefixes.add("CORE.coreName"); @@ -260,9 +263,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter try { SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params); Map m = rsp.nl.asMap(4); - if (requestedTags.contains(DISK)) { + if (requestedTags.contains(FREEDISK.tagName)) { Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.usableSpace"); - if (n != null) ctx.getTags().put(DISK, Suggestion.getTagType(DISK).convertVal(n)); + if (n != null) ctx.getTags().put(FREEDISK.tagName, FREEDISK.convertVal(n)); + } + if (requestedTags.contains(TOTALDISK.tagName)) { + Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.totalSpace"); + if (n != null) ctx.getTags().put(TOTALDISK.tagName, TOTALDISK.convertVal(n)); } if (requestedTags.contains(CORES)) { int count = 0; diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java index 1fba397811e..fc57e3dd1a6 100644 --- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java @@ -107,6 +107,11 @@ public interface MapWriter extends MapSerializable { return this; } + default EntryWriter putStringIfNotNull(String k, Object v) throws IOException { + if(v != null) put(k,String.valueOf(v)); + return this; + } + default EntryWriter put(String k, int v) throws IOException { put(k, (Integer) v); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java index 112d1365101..2d74db87aed 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java @@ -212,30 +212,30 @@ public class TestPolicy extends SolrTestCaseJ4 { assertEquals(Operand.RANGE_EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.01", null)); assertEquals(Operand.RANGE_NOT_EQUAL, REPLICA.getOperand(Operand.NOT_EQUAL, "2.01", null)); - Clause clause = Clause.create((Map) Utils.fromJSONString("{replica: '1.23', node:'#ANY'}")); + Clause clause = Clause.create("{replica: '1.23', node:'#ANY'}"); assertTrue(clause.getReplica().isPass(2)); assertTrue(clause.getReplica().isPass(1)); assertFalse(clause.getReplica().isPass(0)); assertFalse(clause.getReplica().isPass(3)); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '<1.23', node:'#ANY'}")); + clause = Clause.create("{replica: '<1.23', node:'#ANY'}"); assertTrue(clause.getReplica().isPass(1)); assertTrue(clause.getReplica().isPass(0)); assertFalse(clause.getReplica().isPass(2)); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '!1.23', node:'#ANY'}")); + clause = Clause.create("{replica: '!1.23', node:'#ANY'}"); assertFalse(clause.getReplica().isPass(2)); assertFalse(clause.getReplica().isPass(1)); assertTrue(clause.getReplica().isPass(0)); assertTrue(clause.getReplica().isPass(3)); - clause = Clause.create((Map) Utils.fromJSONString("{replica: 1.23, node:'#ANY'}")); + clause = Clause.create("{replica: 1.23, node:'#ANY'}"); assertTrue(clause.getReplica().isPass(2)); assertTrue(clause.getReplica().isPass(1)); assertFalse(clause.getReplica().isPass(0)); assertFalse(clause.getReplica().isPass(3)); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '33%', node:'#ANY'}")); + clause = Clause.create("{replica: '33%', node:'#ANY'}"); assertEquals(Operand.RANGE_EQUAL, clause.getReplica().op); clause = clause.getSealedClause(condition -> { if (condition.name.equals("replica")) { @@ -245,7 +245,7 @@ public class TestPolicy extends SolrTestCaseJ4 { }); assertTrue( clause.getReplica().isPass(2)); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '3 - 5', node:'#ANY'}")); + clause = Clause.create("{replica: '3 - 5', node:'#ANY'}"); assertEquals(Operand.RANGE_EQUAL, clause.getReplica().getOperand()); RangeVal range = (RangeVal) clause.getReplica().getValue(); assertEquals(3.0 , range.min); @@ -261,18 +261,27 @@ public class TestPolicy extends SolrTestCaseJ4 { assertEquals(new Double(0.0), clause.replica.delta(4)); expectThrows(IllegalArgumentException.class, - () -> Clause.create((Map) Utils.fromJSONString("{replica: '-33%', node:'#ANY'}"))); + () -> Clause.create("{replica: '-33%', node:'#ANY'}")); expectThrows(IllegalArgumentException.class, - () -> Clause.create((Map) Utils.fromJSONString("{replica: 'x%', node:'#ANY'}"))); + () -> Clause.create("{replica: 'x%', node:'#ANY'}")); expectThrows(IllegalArgumentException.class, - () -> Clause.create((Map) Utils.fromJSONString("{replica: '20%-33%', node:'#ANY'}"))); + () -> Clause.create("{replica: '20%-33%', node:'#ANY'}")); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}")); + clause = Clause.create("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}"); assertEquals(Operand.RANGE_EQUAL, clause.replica.op); - clause = Clause.create((Map) Utils.fromJSONString("{replica: '#EQUAL', node:'#ANY'}")); + clause = Clause.create("{replica: '#EQUAL', node:'#ANY'}"); assertEquals(Operand.RANGE_EQUAL, clause.replica.op); expectThrows(IllegalArgumentException.class, - () -> Clause.create((Map) Utils.fromJSONString("{replica: '#EQUAL', node:'node_1'}"))); + () -> Clause.create("{replica: '#EQUAL', node:'node_1'}")); + clause = Clause.create("{replica : 0, freedisk:'<20%'}"); + assertEquals(clause.tag.computedType, Clause.ComputedType.PERCENT); + assertEquals(clause.tag.op, Operand.LESS_THAN); + expectThrows(IllegalArgumentException.class, + () -> Clause.create("{replica : 0, INDEX.sizeInGB:'>300'}")); + expectThrows(IllegalArgumentException.class, + () -> Clause.create("{replica:'<3', shard: '#ANV', node:'#ANY'}")); + expectThrows(IllegalArgumentException.class, + () -> Clause.create("{replica:'<3', shard: '#EACH', node:'#E4CH'}")); } @@ -428,36 +437,36 @@ public class TestPolicy extends SolrTestCaseJ4 { } public void testOperands() { - Clause c = Clause.create((Map) Utils.fromJSONString("{replica:'<2', node:'#ANY'}")); + Clause c = Clause.create("{replica:'<2', node:'#ANY'}"); assertFalse(c.replica.isPass(3)); assertFalse(c.replica.isPass(2)); assertTrue(c.replica.isPass(1)); assertEquals("{\"replica\":\"<2.0\"}", c.replica.toString()); - c = Clause.create((Map) Utils.fromJSONString("{replica:'>2', node:'#ANY'}")); + c = Clause.create("{replica:'>2', node:'#ANY'}"); assertTrue(c.replica.isPass(3)); assertFalse(c.replica.isPass(2)); assertFalse(c.replica.isPass(1)); assertEquals("{\"replica\":\">2.0\"}", c.replica.toString()); - c = Clause.create((Map) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}")); + c = Clause.create("{replica:0, nodeRole:'!overseer'}"); assertTrue(c.tag.isPass("OVERSEER")); assertFalse(c.tag.isPass("overseer")); - c = Clause.create((Map) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}")); + c = Clause.create("{replica:0, sysLoadAvg:'<12.7'}"); assertTrue(c.tag.isPass("12.6")); assertTrue(c.tag.isPass(12.6d)); assertFalse(c.tag.isPass("12.9")); assertFalse(c.tag.isPass(12.9d)); - c = Clause.create((Map) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}")); + c = Clause.create("{replica:0, sysLoadAvg:'>12.7'}"); assertTrue(c.tag.isPass("12.8")); assertTrue(c.tag.isPass(12.8d)); assertFalse(c.tag.isPass("12.6")); assertFalse(c.tag.isPass(12.6d)); - c = Clause.create((Map) Utils.fromJSONString("{replica:0, 'metrics:x:y:z':'>12.7'}")); + c = Clause.create("{replica:0, 'metrics:x:y:z':'>12.7'}"); assertTrue(c.tag.val instanceof String); assertTrue(c.tag.isPass("12.8")); assertTrue(c.tag.isPass(12.8d)); @@ -465,7 +474,7 @@ public class TestPolicy extends SolrTestCaseJ4 { assertFalse(c.tag.isPass(12.6d)); try { - c = Clause.create((Map) Utils.fromJSONString("{replica:0, 'ip_1':'<30%'}")); + c = Clause.create("{replica:0, 'ip_1':'<30%'}"); fail("Expected exception"); } catch (Exception e) { assertTrue(e.getMessage().contains("'%' is not allowed for variable : 'ip_1'")); @@ -1099,7 +1108,7 @@ public class TestPolicy extends SolrTestCaseJ4 { Policy.Session session; session = policy.createSession(getSolrCloudManager(nodeValues, clusterState)); - List l = session.getSorted(); + List l = session.getSortedNodes(); assertEquals("node1", l.get(0).node); assertEquals("node3", l.get(1).node); assertEquals("node4", l.get(2).node); @@ -1387,7 +1396,7 @@ public class TestPolicy extends SolrTestCaseJ4 { "{replica:0, 'nodeRole':'overseer','strict':false}," + "{'replica':'<1','node':'node3'}," + "{'replica':'<2','node':'#ANY','shard':'#EACH'}," + - "{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" + + "{'replica':'<3','shard':'#EACH','sysprop.rack':'#EACH'}" + "]" + "}" + "}"; @@ -1871,6 +1880,10 @@ public class TestPolicy extends SolrTestCaseJ4 { List violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations(); assertEquals(1, violations.size()); assertEquals(4, violations.get(0).getViolatingReplicas().size()); + for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { + assertEquals(500d, r.delta, 0.1); + + } List l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata)); assertEquals(3, l.size()); Map m = l.get(0).toMap(new LinkedHashMap<>()); @@ -2873,4 +2886,66 @@ public void testUtilizeNodeFailure2() throws Exception { } + public void testFreediskPercentage(){ + String dataproviderdata = "{" + + " 'liveNodes': [" + + " 'node1:8983'," + + " 'node2:8984'," + + " 'node3:8985'" + + " ]," + + " 'replicaInfo': {" + + " 'node1:8983': {" + + " 'c1': {" + + " 's1': [" + + " {'r1': {'type': 'NRT'}}," + + " {'r2': {'type': 'NRT'}}" + + " ]," + + " 's2': [" + + " {'r1': {'type': 'NRT'}}," + + " {'r2': {'type': 'NRT'}}" + + " ]" + + " }" + + " }" + + " }," + + " 'nodeValues': {" + + " 'node1:8983': {" + + " 'cores': 4," + + " 'freedisk': 230," + + " 'totaldisk': 800," + + " 'port': 8983" + + " }," + + " 'node2:8984': {" + + " 'cores': 0," + + " 'freedisk': 1000," + + " 'totaldisk': 1200," + + " 'port': 8984" + + " }," + + " 'node3:8985': {" + + " 'cores': 0," + + " 'freedisk': 1500," + + " 'totaldisk': 1700," + + " 'port': 8985" + + " }" + + " }" + + "}"; + String autoScalingjson = "{" + + " 'cluster-preferences': [" + + " { 'maximize': 'freedisk', 'precision': 50}," + + " { 'minimize': 'cores', 'precision': 3}" + + " ]," + + " 'cluster-policy': [" + + " { 'replica': 0, freedisk : '<30%'}" + + " ]" + + "}"; + AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); + List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) Utils.fromJSONString(dataproviderdata))).getViolations(); + assertEquals(1, violations.size()); + assertEquals(4, violations.get(0).getViolatingReplicas().size()); + for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { + assertEquals(10.0d, r.delta.doubleValue(), 0.1); + } + + } + + }