mirror of https://github.com/apache/lucene.git
SOLR-12618: AutoScalingHandlerTest failing in jenkins
This commit is contained in:
Noble Paul
parent
2674c53809
commit
9b418a4593
|
|
@ -23,6 +23,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
|
|
@ -675,7 +676,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
|
||||
public void testReadApi() throws Exception {
|
||||
CloudSolrClient solrClient = cluster.getSolrClient();
|
||||
// first trigger
|
||||
|
|
@ -693,9 +693,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
String setClusterPolicyCommand = "{" +
|
||||
" 'set-cluster-policy': [" +
|
||||
" {'cores':'<10', 'node':'#ANY'}," +
|
||||
" {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
|
||||
" {'nodeRole':'overseer', 'replica':0}" +
|
||||
" ]" +
|
||||
" {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" +
|
||||
"}";
|
||||
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
|
||||
response = solrClient.request(req);
|
||||
|
|
@ -713,17 +711,9 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
assertEquals(response.get("result").toString(), "success");
|
||||
|
||||
String setPolicyCommand = "{'set-policy': {" +
|
||||
" 'xyz':[" +
|
||||
" {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
|
||||
" {'nodeRole':'overseer', 'replica':0}" +
|
||||
" ]," +
|
||||
" 'policy1':[" +
|
||||
" {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
|
||||
" ]," +
|
||||
" 'policy2':[" +
|
||||
" {'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}" +
|
||||
" ]" +
|
||||
"}}";
|
||||
" 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
|
||||
" 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
|
||||
" 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}";
|
||||
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand);
|
||||
response = solrClient.request(req);
|
||||
assertEquals(response.get("result").toString(), "success");
|
||||
|
|
@ -747,7 +737,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
|
||||
List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
|
||||
assertNotNull(clusterPolicy);
|
||||
assertEquals(3, clusterPolicy.size());
|
||||
assertEquals(2, clusterPolicy.size());
|
||||
|
||||
Map policies = (Map) response.get("policies");
|
||||
assertNotNull(policies);
|
||||
|
|
@ -766,11 +756,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
for (int i = 0; i < 2; i++) {
|
||||
Map node = (Map) sortedNodes.get(i);
|
||||
assertNotNull(node);
|
||||
assertEquals(6, node.size());
|
||||
assertNotNull(node.get("node"));
|
||||
assertNotNull(node.get("cores"));
|
||||
assertEquals(0d, node.get("cores"));
|
||||
assertNotNull(node.get("freedisk"));
|
||||
assertNotNull(node.get("replicas"));
|
||||
assertTrue(node.get("freedisk") instanceof Double);
|
||||
assertNotNull(node.get("sysLoadAvg"));
|
||||
assertTrue(node.get("sysLoadAvg") instanceof Double);
|
||||
|
|
@ -800,8 +790,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
String tempClusterPolicyCommand = "{" +
|
||||
" 'set-cluster-policy': [" +
|
||||
" {'cores':'<10', 'node':'#ANY'}," +
|
||||
" {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}," +
|
||||
" {'nodeRole':'overseer', 'replica':0}" +
|
||||
" {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+
|
||||
" ]" +
|
||||
"}";
|
||||
req = createAutoScalingRequest(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
|
||||
|
|
@ -831,12 +820,26 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
|
|||
for (Map<String, Object> violation : violations) {
|
||||
assertEquals("readApiTestViolations", violation.get("collection"));
|
||||
assertEquals("shard1", violation.get("shard"));
|
||||
assertEquals(2d, getObjectByPath(violation, true, "violation/delta"));
|
||||
assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta"));
|
||||
assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
|
||||
assertNotNull(violation.get("clause"));
|
||||
}
|
||||
log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners()
|
||||
.stream()
|
||||
.map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList()));
|
||||
JettySolrRunner runner1 = cluster.startJettySolrRunner();
|
||||
cluster.waitForAllNodes(30);
|
||||
log.info("started new jetty {}", runner1.getNodeName());
|
||||
|
||||
response = waitForResponse(namedList -> {
|
||||
List l = (List) Utils.getObjectByPath(namedList, false, "diagnostics/liveNodes");
|
||||
if (l != null && l.contains(runner1.getNodeName())) return true;
|
||||
return false;
|
||||
},
|
||||
createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null),
|
||||
200,
|
||||
20,
|
||||
runner1.getNodeName() + " could not come up ");
|
||||
|
||||
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/suggestions", null);
|
||||
response = solrClient.request(req);
|
||||
|
|
|
|||
|
|
@ -47,16 +47,19 @@ class AddReplicaSuggester extends Suggester {
|
|||
//iterate through nodes and identify the least loaded
|
||||
List<Violation> leastSeriousViolation = null;
|
||||
Row bestNode = null;
|
||||
double[] bestDeviation = null;
|
||||
for (int i = getMatrix().size() - 1; i >= 0; i--) {
|
||||
Row row = getMatrix().get(i);
|
||||
if (!isNodeSuitableForReplicaAddition(row)) continue;
|
||||
Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict);
|
||||
List<Violation> errs = testChangedMatrix(strict, tmpRow.session);
|
||||
|
||||
double[] deviation = new double[1];
|
||||
List<Violation> errs = testChangedMatrix(strict, tmpRow.session, deviation);
|
||||
if (!containsNewErrors(errs)) {
|
||||
if (isLessSerious(errs, leastSeriousViolation)) {
|
||||
if ((errs.isEmpty() && isLessDeviant(bestDeviation, deviation)) ||//there are no violations but this is deviating less
|
||||
isLessSerious(errs, leastSeriousViolation)) {//there are errors , but this has less serious violation
|
||||
leastSeriousViolation = errs;
|
||||
bestNode = tmpRow;
|
||||
bestDeviation = deviation;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -73,6 +76,7 @@ class AddReplicaSuggester extends Suggester {
|
|||
return null;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public CollectionParams.CollectionAction getAction() {
|
||||
return ADDREPLICA;
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
|
|||
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
|
||||
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
|
||||
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
|
||||
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL;
|
||||
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
|
||||
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
|
||||
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
|
||||
|
|
@ -215,7 +216,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
|
|||
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
|
||||
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
|
||||
if (v == 0) {// higher the number of replicas , harder to satisfy
|
||||
v = Preference.compareWithTolerance((Double) this.replica.val, (Double) that.replica.val, 1);
|
||||
Double thisVal = this.replica.val instanceof RangeVal ? ((RangeVal) this.replica.val).max.doubleValue() : (Double) this.replica.val;
|
||||
Double thatVal = that.replica.val instanceof RangeVal ? ((RangeVal) that.replica.val).max.doubleValue() : (Double) that.replica.val;
|
||||
v = Preference.compareWithTolerance(thisVal, thatVal, 1);
|
||||
v = this.replica.op == LESS_THAN ? v : v * -1;
|
||||
}
|
||||
if (v == 0) v = compareTypes(this.type, that.type);
|
||||
|
|
@ -240,7 +243,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
|
|||
|
||||
//replica value is zero
|
||||
boolean isReplicaZero() {
|
||||
return replica != null && replica.getOperand() == Operand.EQUAL &&
|
||||
return replica != null && replica.getOperand() == Operand.EQUAL && replica.val instanceof Double &&
|
||||
Preference.compareWithTolerance(0d, (Double) replica.val, 1) == 0;
|
||||
}
|
||||
|
||||
|
|
@ -353,7 +356,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
|
|||
return operand;
|
||||
}
|
||||
|
||||
public List<Violation> test(Policy.Session session) {
|
||||
public List<Violation> test(Policy.Session session, double[] deviations) {
|
||||
ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
|
||||
Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
|
||||
if (isPerCollectiontag()) {
|
||||
|
|
@ -377,6 +380,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
|
|||
sealedClause.getReplica().delta(replicas),
|
||||
tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
|
||||
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
|
||||
} else {
|
||||
if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
|
||||
Number actualCount = replicas.getVal(type);
|
||||
Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
|
||||
realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
|
||||
deviations[0] += Math.abs(realDelta);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -422,7 +432,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
|
|||
tagVsCount.get(row.node).increment(shards.getValue());
|
||||
}
|
||||
} else {
|
||||
tagVsCount.computeIfAbsent(String.valueOf(t.getValue()), s -> new ReplicaCount());
|
||||
boolean pass = sealedClause.getTag().isPass(tagVal);
|
||||
if(!pass && !isReplicaZero()) continue;
|
||||
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
|
||||
if (pass) {
|
||||
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
|
||||
|
|
|
|||
|
|
@ -62,7 +62,8 @@ public class MoveReplicaSuggester extends Suggester {
|
|||
if (!isNodeSuitableForReplicaAddition(targetRow)) continue;
|
||||
targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
|
||||
Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
|
||||
List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
|
||||
double[] deviation = new double[1];
|
||||
List<Violation> errs = testChangedMatrix(strict, srcRowModified.session, deviation);
|
||||
srcRowModified.session.applyRules(); // now resort the nodes with the new values
|
||||
Policy.Session tmpSession = srcRowModified.session;
|
||||
|
||||
|
|
|
|||
|
|
@ -33,13 +33,12 @@ public class NodeVariable extends VariableBase {
|
|||
if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
|
||||
for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
|
||||
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
|
||||
.forceOperation(true)
|
||||
.hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
|
||||
.hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
|
||||
ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
|
||||
ctx.addSuggestion(suggester);
|
||||
if(ctx.addSuggestion(suggester) == null) break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -572,7 +572,7 @@ public class Policy implements MapWriter {
|
|||
setApproxValuesAndSortNodes(clusterPreferences, matrix);
|
||||
|
||||
for (Clause clause : expandedClauses) {
|
||||
List<Violation> errs = clause.test(this);
|
||||
List<Violation> errs = clause.test(this, null);
|
||||
violations.addAll(errs);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,12 +28,14 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.cloud.DistribStateManager;
|
||||
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
|
||||
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
|
||||
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
|
||||
import org.apache.solr.common.IteratorWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
|
|
@ -183,30 +185,20 @@ public class PolicyHelper {
|
|||
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
|
||||
Policy.Session session = policy.createSession(cloudManager);
|
||||
List<Row> sorted = session.getSortedNodes();
|
||||
List<Violation> violations = session.getViolations();
|
||||
|
||||
List<Preference> clusterPreferences = policy.getClusterPreferences();
|
||||
|
||||
List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
|
||||
for (Row row : sorted) {
|
||||
Map<String, Object> map = Utils.makeMap("node", row.node);
|
||||
map.put("isLive", row.isLive);
|
||||
for (Cell cell : row.getCells()) {
|
||||
for (Preference clusterPreference : clusterPreferences) {
|
||||
Policy.SortParam name = clusterPreference.getName();
|
||||
if (cell.getName().equalsIgnoreCase(name.name())) {
|
||||
map.put(name.name(), cell.getValue());
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ew -> ew.put("sortedNodes", (IteratorWriter) iw -> {
|
||||
for (Row row : sorted) {
|
||||
iw.add((MapWriter) ew1 -> {
|
||||
ew1.put("node", row.node).
|
||||
put("isLive", row.isLive);
|
||||
for (Cell cell : row.getCells())
|
||||
ew1.put(cell.name, cell.val,
|
||||
(Predicate) o -> o != null && (!(o instanceof Map) || !((Map) o).isEmpty()));
|
||||
ew1.put("replicas", row.collectionVsShardVsReplicas);
|
||||
});
|
||||
}
|
||||
sortedNodes.add(map);
|
||||
}
|
||||
|
||||
return ew -> {
|
||||
ew.put("sortedNodes", sortedNodes);
|
||||
ew.put("violations", violations);
|
||||
};
|
||||
}).put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes())
|
||||
.put("violations", session.getViolations())
|
||||
.put("config", session.getPolicy());
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,8 +35,12 @@ class RangeVal implements MapWriter {
|
|||
Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
|
||||
}
|
||||
|
||||
public Double realDelta(double v) {
|
||||
if (actual != null) return v - actual.doubleValue();
|
||||
else return delta(v);
|
||||
}
|
||||
|
||||
public Double delta(double v) {
|
||||
// if (actual != null) return v - actual.doubleValue();
|
||||
if (v >= max.doubleValue()) return v - max.doubleValue();
|
||||
if (v <= min.doubleValue()) return v - min.doubleValue();
|
||||
return 0d;
|
||||
|
|
@ -49,6 +53,8 @@ class RangeVal implements MapWriter {
|
|||
|
||||
@Override
|
||||
public void writeMap(EntryWriter ew) throws IOException {
|
||||
ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
|
||||
ew.put("min", min)
|
||||
.put("max", max)
|
||||
.putIfNotNull("actual", actual);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ class ReplicaCount implements MapWriter {
|
|||
|
||||
@Override
|
||||
public void writeMap(EntryWriter ew) throws IOException {
|
||||
ew.put(Replica.Type.NRT.name(), nrt);
|
||||
ew.put(Replica.Type.PULL.name(), pull);
|
||||
ew.put(Replica.Type.TLOG.name(), tlog);
|
||||
if (nrt > 0) ew.put(Replica.Type.NRT.name(), nrt);
|
||||
if (pull > 0) ew.put(Replica.Type.PULL.name(), pull);
|
||||
if (tlog > 0) ew.put(Replica.Type.TLOG.name(), tlog);
|
||||
ew.put("count", total());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ public class ReplicaInfo implements MapWriter {
|
|||
}
|
||||
|
||||
public ReplicaInfo(String name, String core, String coll, String shard, Replica.Type type, String node, Map<String, Object> vals) {
|
||||
if(vals==null) vals = Collections.emptyMap();
|
||||
if (vals == null) vals = Collections.emptyMap();
|
||||
this.name = name;
|
||||
if (vals != null) {
|
||||
this.variables.putAll(vals);
|
||||
|
|
@ -66,6 +66,21 @@ public class ReplicaInfo implements MapWriter {
|
|||
this.node = node;
|
||||
}
|
||||
|
||||
ReplicaInfo(Map<String, Object> map) {
|
||||
this.name = map.keySet().iterator().next();
|
||||
Map details = (Map) map.get(name);
|
||||
details = Utils.getDeepCopy(details, 4);
|
||||
this.collection = (String) details.remove("collection");
|
||||
this.shard = (String) details.remove("shard");
|
||||
this.core = (String) details.remove("core");
|
||||
this.node = (String) details.remove("node_name");
|
||||
this.isLeader = Boolean.parseBoolean((String) details.getOrDefault("leader", "false"));
|
||||
details.remove("leader");
|
||||
type = Replica.Type.valueOf((String) details.getOrDefault("type", "NRT"));
|
||||
details.remove("type");
|
||||
this.variables.putAll(details);
|
||||
}
|
||||
|
||||
public Object clone() {
|
||||
return new ReplicaInfo(name, core, collection, shard, type, node, variables);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import java.util.function.Predicate;
|
|||
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
|
||||
import org.apache.solr.common.ConditionalMapWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
|
|
@ -70,6 +71,10 @@ public abstract class Suggester implements MapWriter {
|
|||
this.session = session.copy();
|
||||
}
|
||||
|
||||
boolean isLessDeviant(double[] previousBest, double[] newDeviation) {
|
||||
if (previousBest == null) return true;
|
||||
return newDeviation[0] < previousBest[0];
|
||||
}
|
||||
public Suggester hint(Hint hint, Object value) {
|
||||
hint.validator.accept(value);
|
||||
if (hint.multiValued) {
|
||||
|
|
@ -203,7 +208,9 @@ public abstract class Suggester implements MapWriter {
|
|||
@Override
|
||||
public void writeMap(EntryWriter ew) throws IOException {
|
||||
ew.put("type", violation == null ? "improvement" : "violation");
|
||||
ew.putIfNotNull("violation", violation);
|
||||
ew.putIfNotNull("violation",
|
||||
new ConditionalMapWriter(violation,
|
||||
(k, v) -> !"violatingReplicas".equals(k)));
|
||||
ew.put("operation", operation);
|
||||
}
|
||||
|
||||
|
|
@ -237,6 +244,7 @@ public abstract class Suggester implements MapWriter {
|
|||
|
||||
boolean containsNewErrors(List<Violation> violations) {
|
||||
boolean isTxOpen = session.transaction != null && session.transaction.isOpen();
|
||||
if (violations.size() > originalViolations.size()) return true;
|
||||
for (Violation v : violations) {
|
||||
//the computed value can change over time. So it's better to evaluate it in the end
|
||||
if (isTxOpen && v.getClause().hasComputedValue) continue;
|
||||
|
|
@ -272,12 +280,12 @@ public abstract class Suggester implements MapWriter {
|
|||
}
|
||||
}
|
||||
|
||||
List<Violation> testChangedMatrix(boolean strict, Policy.Session session) {
|
||||
List<Violation> testChangedMatrix(boolean strict, Policy.Session session, double[] deviation) {
|
||||
Policy.setApproxValuesAndSortNodes(session.getPolicy().clusterPreferences, session.matrix);
|
||||
List<Violation> errors = new ArrayList<>();
|
||||
for (Clause clause : session.expandedClauses) {
|
||||
if (strict || clause.strict) {
|
||||
List<Violation> errs = clause.test(session);
|
||||
List<Violation> errs = clause.test(session, deviation);
|
||||
if (!errs.isEmpty()) {
|
||||
errors.addAll(errs);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ public class VariableBase implements Variable {
|
|||
val = condition.op.readRuleValue(condition);
|
||||
if (val != condition.val) return val;
|
||||
}
|
||||
if (!isRuleVal && "".equals(val) && this.varType.type != String.class) val = -1;
|
||||
if (name == null) name = this.varType.tagName;
|
||||
if (varType.type == Double.class) {
|
||||
Double num = Clause.parseDouble(name, val);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import java.util.List;
|
|||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.solr.common.IteratorWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
|
||||
|
|
@ -160,13 +161,20 @@ public class Violation implements MapWriter {
|
|||
ew.putIfNotNull("collection", coll);
|
||||
ew.putIfNotNull("shard", shard);
|
||||
ew.putIfNotNull("node", node);
|
||||
ew.putIfNotNull("tagKey", String.valueOf(tagKey));
|
||||
ew.putStringIfNotNull("tagKey", tagKey);
|
||||
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
|
||||
if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
|
||||
else ew1.put(clause.tag.name, String.valueOf(actualVal));
|
||||
ew1.putIfNotNull("delta", replicaCountDelta);
|
||||
});
|
||||
ew.put("clause", getClause());
|
||||
if (!replicaInfoAndErrs.isEmpty()) {
|
||||
ew.put("violatingReplicas", (IteratorWriter) iw -> {
|
||||
for (ReplicaInfoAndErr replicaInfoAndErr : replicaInfoAndErrs) {
|
||||
iw.add(replicaInfoAndErr.replicaInfo);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
static class Ctx {
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
|
|||
return solrClient.getClusterStateProvider();
|
||||
}
|
||||
|
||||
private void readReplicaDetails() throws IOException {
|
||||
protected void readReplicaDetails() throws IOException {
|
||||
ClusterStateProvider clusterStateProvider = getClusterStateProvider();
|
||||
ClusterState clusterState = clusterStateProvider.getClusterState();
|
||||
if (clusterState == null) { // zkStateReader still initializing
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.common;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.BiPredicate;
|
||||
|
||||
public class ConditionalMapWriter implements MapWriter {
|
||||
private final MapWriter delegate;
|
||||
private final BiPredicate<String, Object> predicate;
|
||||
|
||||
public ConditionalMapWriter(MapWriter delegate, BiPredicate<String, Object> predicate) {
|
||||
this.delegate = delegate;
|
||||
this.predicate = predicate;
|
||||
}
|
||||
|
||||
private class EntryWriterWrapper implements EntryWriter {
|
||||
private final EntryWriter delegate;
|
||||
|
||||
EntryWriterWrapper(EntryWriter delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, Object v) throws IOException {
|
||||
if (predicate.test(k, v)) delegate.put(k, v);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, int v) throws IOException {
|
||||
return put(k, Integer.valueOf(v));
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, long v) throws IOException {
|
||||
return put(k, Long.valueOf(v));
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, float v) throws IOException {
|
||||
return put(k, Float.valueOf(v));
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, double v) throws IOException {
|
||||
return put(k, Double.valueOf(v));
|
||||
}
|
||||
|
||||
@Override
|
||||
public EntryWriter put(String k, boolean v) throws IOException {
|
||||
return put(k, Boolean.valueOf(v));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeMap(EntryWriter ew) throws IOException {
|
||||
delegate.writeMap(new EntryWriterWrapper(ew));
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ import java.util.ArrayList;
|
|||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import org.apache.solr.common.util.Utils;
|
||||
|
||||
|
|
@ -102,6 +103,11 @@ public interface MapWriter extends MapSerializable {
|
|||
return this;
|
||||
}
|
||||
|
||||
default EntryWriter put(String k, Object v, Predicate<Object> p) throws IOException {
|
||||
if (p.test(v)) put(k, v);
|
||||
return this;
|
||||
}
|
||||
|
||||
default EntryWriter putIfNotNull(String k, Object v) throws IOException {
|
||||
if(v != null) put(k,v);
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -2467,8 +2467,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
|
|||
" }}}";
|
||||
|
||||
String autoScalingjson = " { cluster-policy:[" +
|
||||
" { replica :'<34%', shard: '#EACH', sysprop.az : east}," +
|
||||
" { replica :'<67%', shard: '#EACH', sysprop.az : west}" +
|
||||
" { replica :'33%', shard: '#EACH', sysprop.az : east}," +
|
||||
" { replica :'67%', shard: '#EACH', sysprop.az : west}" +
|
||||
" ]," +
|
||||
" cluster-preferences :[{ minimize : cores }]}";
|
||||
|
||||
|
|
@ -2480,6 +2480,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
|
|||
|
||||
List<String> nodes = new ArrayList<>();
|
||||
|
||||
int westCount = 0, eastCount = 0;
|
||||
for (int i = 0; i < 12; i++) {
|
||||
SolrRequest suggestion = txn.getCurrentSession()
|
||||
.getSuggester(ADDREPLICA)
|
||||
|
|
@ -2488,9 +2489,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
|
|||
assertNotNull(suggestion);
|
||||
String node = suggestion.getParams().get("node");
|
||||
nodes.add(node);
|
||||
if ("10.0.0.6:8983_solr".equals(node)) eastCount++;
|
||||
if ("10.0.0.6:7574_solr".equals(node)) westCount++;
|
||||
if (i % 3 == 1) assertEquals("10.0.0.6:8983_solr", node);
|
||||
else assertEquals("10.0.0.6:7574_solr", node);
|
||||
}
|
||||
assertEquals(8, westCount);
|
||||
assertEquals(4, eastCount);
|
||||
|
||||
List<Violation> violations = txn.close();
|
||||
assertTrue(violations.isEmpty());
|
||||
|
|
@ -2655,7 +2660,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
|
|||
"}";
|
||||
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
|
||||
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
|
||||
assertEquals(2, violations.size());
|
||||
assertEquals("expected 2 violations", 2, violations.size());
|
||||
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
|
||||
assertEquals(2, suggestions.size());
|
||||
for (Suggester.SuggestionInfo suggestion : suggestions) {
|
||||
|
|
@ -3562,11 +3567,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
|
|||
Object root = Utils.fromJSONString(writer.toString());
|
||||
assertEquals(2l,
|
||||
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/NRT"));
|
||||
assertEquals(0l,
|
||||
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/PULL"));
|
||||
assertEquals(0l,
|
||||
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/TLOG"));
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
|
@ -288,4 +289,243 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
|
|||
};
|
||||
}
|
||||
|
||||
public void testAutoScalingHandlerFailure() {
|
||||
String diagnostics = "{" +
|
||||
" 'diagnostics': {" +
|
||||
" 'sortedNodes': [" +
|
||||
" {" +
|
||||
" 'node': '127.0.0.1:63191_solr'," +
|
||||
" 'isLive': true," +
|
||||
" 'cores': 3.0," +
|
||||
" 'freedisk': 680.908073425293," +
|
||||
" 'heapUsage': 24.97510064011647," +
|
||||
" 'sysLoadAvg': 272.75390625," +
|
||||
" 'totaldisk': 1037.938980102539," +
|
||||
" 'replicas': {" +
|
||||
" 'readApiTestViolations': {" +
|
||||
" 'shard1': [" +
|
||||
" {" +
|
||||
" 'core_node5': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n2'," +
|
||||
" 'leader': 'true'," +
|
||||
" 'base_url': 'https://127.0.0.1:63191/solr'," +
|
||||
" 'node_name': '127.0.0.1:63191_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'core_node7': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n4'," +
|
||||
" 'base_url': 'https://127.0.0.1:63191/solr'," +
|
||||
" 'node_name': '127.0.0.1:63191_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'core_node12': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n10'," +
|
||||
" 'base_url': 'https://127.0.0.1:63191/solr'," +
|
||||
" 'node_name': '127.0.0.1:63191_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }" +
|
||||
" ]" +
|
||||
" }" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'node': '127.0.0.1:63192_solr'," +
|
||||
" 'isLive': true," +
|
||||
" 'cores': 3.0," +
|
||||
" 'freedisk': 680.908073425293," +
|
||||
" 'heapUsage': 24.98878807983566," +
|
||||
" 'sysLoadAvg': 272.75390625," +
|
||||
" 'totaldisk': 1037.938980102539," +
|
||||
" 'replicas': {" +
|
||||
" 'readApiTestViolations': {" +
|
||||
" 'shard1': [" +
|
||||
" {" +
|
||||
" 'core_node3': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n1'," +
|
||||
" 'base_url': 'https://127.0.0.1:63192/solr'," +
|
||||
" 'node_name': '127.0.0.1:63192_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'core_node9': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n6'," +
|
||||
" 'base_url': 'https://127.0.0.1:63192/solr'," +
|
||||
" 'node_name': '127.0.0.1:63192_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'core_node11': {" +
|
||||
" 'core': 'readApiTestViolations_shard1_replica_n8'," +
|
||||
" 'base_url': 'https://127.0.0.1:63192/solr'," +
|
||||
" 'node_name': '127.0.0.1:63192_solr'," +
|
||||
" 'state': 'active'," +
|
||||
" 'type': 'NRT'," +
|
||||
" 'force_set_state': 'false'," +
|
||||
" 'INDEX.sizeInGB': 6.426125764846802E-8," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }" +
|
||||
" ]" +
|
||||
" }" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'node': '127.0.0.1:63219_solr'," +
|
||||
" 'isLive': true," +
|
||||
" 'cores': 0.0," +
|
||||
" 'freedisk': 680.908073425293," +
|
||||
" 'heapUsage': 24.98878807983566," +
|
||||
" 'sysLoadAvg': 272.75390625," +
|
||||
" 'totaldisk': 1037.938980102539," +
|
||||
" 'replicas': {}" +
|
||||
" }" +
|
||||
" ]," +
|
||||
" 'liveNodes': [" +
|
||||
" '127.0.0.1:63191_solr'," +
|
||||
" '127.0.0.1:63192_solr'," +
|
||||
" '127.0.0.1:63219_solr'" +
|
||||
" ]," +
|
||||
" 'violations': [" +
|
||||
" {" +
|
||||
" 'collection': 'readApiTestViolations'," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'node': '127.0.0.1:63191_solr'," +
|
||||
" 'violation': {" +
|
||||
" 'replica': {'NRT': 3, 'count': 3}," +
|
||||
" 'delta': 2.0" +
|
||||
" }," +
|
||||
" 'clause': {" +
|
||||
" 'replica': '<3'," +
|
||||
" 'shard': '#EACH'," +
|
||||
" 'node': '#ANY'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }," +
|
||||
" {" +
|
||||
" 'collection': 'readApiTestViolations'," +
|
||||
" 'shard': 'shard1'," +
|
||||
" 'node': '127.0.0.1:63192_solr'," +
|
||||
" 'violation': {" +
|
||||
" 'replica': {'NRT': 3, 'count': 3}," +
|
||||
" 'delta': 2.0" +
|
||||
" }," +
|
||||
" 'clause': {" +
|
||||
" 'replica': '<2'," +
|
||||
" 'shard': '#EACH'," +
|
||||
" 'node': '#ANY'," +
|
||||
" 'collection': 'readApiTestViolations'" +
|
||||
" }" +
|
||||
" }" +
|
||||
" ]," +
|
||||
" 'config': {" +
|
||||
" 'cluster-preferences': [" +
|
||||
" {'minimize': 'cores', 'precision': 3}," +
|
||||
" {'maximize': 'freedisk', 'precision': 100}," +
|
||||
" {'minimize': 'sysLoadAvg', 'precision': 10}," +
|
||||
" {'minimize': 'heapUsage', 'precision': 10}" +
|
||||
" ]," +
|
||||
" 'cluster-policy': [" +
|
||||
" {'cores': '<10', 'node': '#ANY'}," +
|
||||
" {'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
|
||||
" {'nodeRole': 'overseer', 'replica': 0}" +
|
||||
" ]" +
|
||||
" }" +
|
||||
" }}";
|
||||
Map<String, Object> m = (Map<String, Object>) Utils.fromJSONString(diagnostics);
|
||||
|
||||
Policy policy = new Policy((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
|
||||
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
|
||||
Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
|
||||
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
|
||||
assertEquals(2, suggestions.size());
|
||||
|
||||
}
|
||||
|
||||
static SolrCloudManager createCloudManagerFromDiagnostics(Map<String, Object> m) {
|
||||
List<Map> sortedNodes = (List<Map>) Utils.getObjectByPath(m, false, "diagnostics/sortedNodes");
|
||||
Set<String> liveNodes = new HashSet<>();
|
||||
SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) {
|
||||
@Override
|
||||
protected void readReplicaDetails() {
|
||||
for (Object o : sortedNodes) {
|
||||
String node = (String) ((Map) o).get("node");
|
||||
liveNodes.add(node);
|
||||
Map nodeDetails = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> new LinkedHashMap<>());
|
||||
Map<String, Map<String, List<Map>>> replicas = (Map<String, Map<String, List<Map>>>) ((Map) o).get("replicas");
|
||||
replicas.forEach((coll, shardVsReplicas) -> shardVsReplicas
|
||||
.forEach((shard, repDetails) -> {
|
||||
List<ReplicaInfo> reps = (List) ((Map) nodeDetails
|
||||
.computeIfAbsent(coll, o1 -> new LinkedHashMap<>()))
|
||||
.computeIfAbsent(shard, o12 -> new ArrayList<ReplicaInfo>());
|
||||
for (Map map : repDetails) reps.add(new ReplicaInfo(map));
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
|
||||
return nodeVsCollectionVsShardVsReplicaInfo.get(node) == null ?
|
||||
Collections.emptyMap() :
|
||||
nodeVsCollectionVsShardVsReplicaInfo.get(node);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
|
||||
for (Map n : sortedNodes) if (n.get("node").equals(node)) return n;
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
};
|
||||
return new DelegatingCloudManager(null) {
|
||||
@Override
|
||||
public NodeStateProvider getNodeStateProvider() {
|
||||
return nodeStateProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterStateProvider getClusterStateProvider() {
|
||||
return new DelegatingClusterStateProvider(null) {
|
||||
@Override
|
||||
public Set<String> getLiveNodes() {
|
||||
return liveNodes;
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
|
|||
import java.util.function.Predicate;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.embedded.JettyConfig;
|
||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||
|
|
@ -49,6 +50,7 @@ import org.apache.solr.common.cloud.Replica;
|
|||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
|
||||
|
|
@ -352,4 +354,21 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
protected NamedList waitForResponse(Predicate<NamedList> predicate, SolrRequest request, int intervalInMillis, int numRetries, String messageOnFail) {
|
||||
int i = 0;
|
||||
for (; i < numRetries; i++) {
|
||||
try {
|
||||
NamedList<Object> response = cluster.getSolrClient().request(request);
|
||||
if (predicate.test(response)) return response;
|
||||
Thread.sleep(intervalInMillis);
|
||||
} catch (RuntimeException rte) {
|
||||
throw rte;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("error executing request", e);
|
||||
}
|
||||
}
|
||||
fail("Tried " + i + " times , could not succeed. " + messageOnFail);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue