Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11997: Suggestions API/UI should show an entry where a violation could not be resolved

This commit is contained in:
Noble Paul 2018-11-06 22:27:26 +11:00
parent 13a83564bb
commit 08fcce4c98
11 changed files with 289 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -92,6 +92,8 @@ New Features
* SOLR-12938: Cluster Status returns results for aliases, instead of throwing exceptions (Gus Heck)
* SOLR-11997: Suggestions API/UI should show an entry where a violation could not be resolved (noble)
Other Changes
----------------------

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
View File

@ -49,7 +49,7 @@ class AddReplicaSuggester extends Suggester {
Row bestNode = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isNodeSuitableForReplicaAddition(row)) continue;
if (!isNodeSuitableForReplicaAddition(row, null)) continue;
Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict);
List<Violation> errs = testChangedMatrix(strict, tmpRow.session);
if (!containsNewErrors(errs)) {

3
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
View File

@ -127,6 +127,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
hasComputedValue = hasComputedValue();
}
public Condition getThirdTag() {
return globalTag == null ? tag : globalTag;
}
private void doPostValidate(Condition... conditions) {
for (Condition condition : conditions) {
if (condition == null) continue;

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
View File

@ -60,7 +60,7 @@ public class MoveReplicaSuggester extends Suggester {
for (int j = session.matrix.size() - 1; j >= stopAt; j--) {
targetRow = session.matrix.get(j);
if (targetRow.node.equals(fromRow.node)) continue;
if (!isNodeSuitableForReplicaAddition(targetRow)) continue;
if (!isNodeSuitableForReplicaAddition(targetRow, fromRow)) continue;
targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);

16
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
View File

@ -237,14 +237,20 @@ public class PolicyHelper {
ctx.session = policy.createSession(cloudManager);
List<Violation> violations = ctx.session.getViolations();
for (Violation violation : violations) {
String name = violation.getClause().isPerCollectiontag() ?
violation.getClause().tag.name :
violation.getClause().globalTag.name;
Variable.Type tagType = VariableBase.getTagType(name);
tagType.getSuggestions(ctx.setViolation(violation));
violation.getClause().getThirdTag().varType.getSuggestions(ctx.setViolation(violation));
ctx.violation = null;
}
for (Violation current : ctx.session.getViolations()) {
for (Violation old : violations) {
if (current.equals(old)) {
//could not be resolved
ctx.suggestions.add(new Suggester.SuggestionInfo(current, null, "unresolved-violation"));
break;
}
}
}
if (ctx.needMore()) {
try {
addMissingReplicas(cloudManager, ctx);

24
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
View File

@ -121,10 +121,26 @@ public abstract class Suggester implements MapWriter {
return this;
}
protected boolean isNodeSuitableForReplicaAddition(Row row) {
if (!row.isLive) return false;
if (!isAllowed(row.node, Hint.TARGET_NODE)) return false;
if (!isAllowed(row.getVal(ImplicitSnitch.DISK), Hint.MINFREEDISK)) return false;
protected boolean isNodeSuitableForReplicaAddition(Row targetRow, Row srcRow) {
if (!targetRow.isLive) return false;
if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) return false;
if (!isAllowed(targetRow.getVal(ImplicitSnitch.DISK), Hint.MINFREEDISK)) return false;
if (srcRow != null) {// if the src row has the same violation it's not
for (Violation v1 : originalViolations) {
if (!v1.getClause().getThirdTag().varType.meta.isNodeSpecificVal()) continue;
if (v1.getClause().hasComputedValue) continue;
if (targetRow.node.equals(v1.node)) {
for (Violation v2 : originalViolations) {
if (srcRow.node.equals(v2.node)) {
if (v1.getClause().equals(v2.getClause()))
return false;
}
}
}
}
}
return true;
}

2
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
View File

@ -35,7 +35,7 @@ public class Suggestion {
int max = Integer.MAX_VALUE;
public Policy.Session session;
public Violation violation;
private List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
SolrRequest addSuggestion(Suggester suggester) {
return addSuggestion(suggester, "violation");
}

212
solr/solrj/src/test-files/solrj/solr/autoscaling/testUnresolvedSuggestion.json Normal file
View File

@ -0,0 +1,212 @@
{
"diagnostics": {
"sortedNodes": [
{
"node": "10.0.0.80:7574_solr",
"isLive": true,
"cores": 2.0,
"freedisk": 661.6284255981445,
"totaldisk": 1037.938980102539,
"replicas": {
"go": {
"shard2": [
{
"core_node7": {
"core": "go_shard2_replica_n4",
"shard": "shard2",
"collection": "go",
"node_name": "10.0.0.80:7574_solr",
"type": "NRT",
"base_url": "http://10.0.0.80:7574/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
},
{
"core_node9": {
"core": "go_shard2_replica_n6",
"shard": "shard2",
"collection": "go",
"node_name": "10.0.0.80:7574_solr",
"type": "NRT",
"leader": "true",
"base_url": "http://10.0.0.80:7574/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
}
]
}
}
},
{
"node": "10.0.0.80:8983_solr",
"isLive": true,
"cores": 2.0,
"freedisk": 661.6284217834473,
"totaldisk": 1037.938980102539,
"replicas": {
"go": {
"shard3": [
{
"core_node11": {
"core": "go_shard3_replica_n8",
"shard": "shard3",
"collection": "go",
"node_name": "10.0.0.80:8983_solr",
"type": "NRT",
"leader": "true",
"base_url": "http://10.0.0.80:8983/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
}
],
"shard1": [
{
"core_node5": {
"core": "go_shard1_replica_n2",
"shard": "shard1",
"collection": "go",
"node_name": "10.0.0.80:8983_solr",
"type": "NRT",
"leader": "true",
"base_url": "http://10.0.0.80:8983/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
}
]
}
}
},
{
"node": "10.0.0.80:8984_solr",
"isLive": true,
"cores": 2.0,
"freedisk": 661.6284217834473,
"totaldisk": 1037.938980102539,
"replicas": {
"go": {
"shard3": [
{
"core_node12": {
"core": "go_shard3_replica_n10",
"shard": "shard3",
"collection": "go",
"node_name": "10.0.0.80:8984_solr",
"type": "NRT",
"base_url": "http://10.0.0.80:8984/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
}
],
"shard1": [
{
"core_node3": {
"core": "go_shard1_replica_n1",
"shard": "shard1",
"collection": "go",
"node_name": "10.0.0.80:8984_solr",
"type": "NRT",
"base_url": "http://10.0.0.80:8984/solr",
"state": "active",
"force_set_state": "false",
"INDEX.sizeInGB": 6.426125764846802E-8
}
}
]
}
}
}
],
"liveNodes": [
"10.0.0.80:7574_solr",
"10.0.0.80:8983_solr",
"10.0.0.80:8984_solr"
],
"config": {
"cluster-preferences": [{"minimize": "cores"}],
"cluster-policy": [{"replica": "<2", "node": "#ANY"}]
}
},
"cluster":{
"collections":{
"go":{
"pullReplicas":"0",
"replicationFactor":"2",
"shards":{
"shard1":{
"range":"80000000-d554ffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"go_shard1_replica_n1",
"base_url":"http://10.0.0.80:8984/solr",
"node_name":"10.0.0.80:8984_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false"},
"core_node5":{
"core":"go_shard1_replica_n2",
"base_url":"http://10.0.0.80:8983/solr",
"node_name":"10.0.0.80:8983_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false",
"leader":"true"}}},
"shard2":{
"range":"d5550000-2aa9ffff",
"state":"active",
"replicas":{
"core_node7":{
"core":"go_shard2_replica_n4",
"base_url":"http://10.0.0.80:7574/solr",
"node_name":"10.0.0.80:7574_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false"},
"core_node9":{
"core":"go_shard2_replica_n6",
"base_url":"http://10.0.0.80:7574/solr",
"node_name":"10.0.0.80:7574_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false",
"leader":"true"}}},
"shard3":{
"range":"2aaa0000-7fffffff",
"state":"active",
"replicas":{
"core_node11":{
"core":"go_shard3_replica_n8",
"base_url":"http://10.0.0.80:8983/solr",
"node_name":"10.0.0.80:8983_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false",
"leader":"true"},
"core_node12":{
"core":"go_shard3_replica_n10",
"base_url":"http://10.0.0.80:8984/solr",
"node_name":"10.0.0.80:8984_solr",
"state":"active",
"type":"NRT",
"force_set_state":"false"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"-1",
"autoAddReplicas":"false",
"nrtReplicas":"2",
"tlogReplicas":"0",
"znodeVersion":9,
"configName":"go"}},
"live_nodes":["10.0.0.80:8983_solr",
"10.0.0.80:7574_solr",
"10.0.0.80:8984_solr"]}
}

47
solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
View File

@ -49,6 +49,7 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.EMPTY_MAP;
import static java.util.Collections.emptyMap;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
import static org.apache.solr.common.util.Utils.getObjectByPath;
public class TestPolicy2 extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -200,7 +201,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
for (String tag : tags) {
if (tag.equals(CORES.tagName))
result.put(CORES.tagName, coreCount.getOrDefault(node, new AtomicInteger(0)).get());
result.put(tag, Utils.getObjectByPath(nodeVals, true, Arrays.asList(node, tag)));
result.put(tag, getObjectByPath(nodeVals, true, Arrays.asList(node, tag)));
}
return result;
}
@ -227,19 +228,20 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
};
}
public void testAutoScalingHandlerFailure() throws IOException {
public void testAutoScalingHandlerFailure() {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testAutoScalingHandlerFailure.json");
Policy policy = new Policy((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
Policy policy = new Policy((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"));
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
assertEquals(2, suggestions.size());
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
// System.out.println(Utils.writeJson(suggestions, new StringWriter(),true).toString());
assertEquals(4, suggestions.size());
}
static SolrCloudManager createCloudManagerFromDiagnostics(Map<String, Object> m) {
List<Map> sortedNodes = (List<Map>) Utils.getObjectByPath(m, false, "diagnostics/sortedNodes");
List<Map> sortedNodes = (List<Map>) getObjectByPath(m, false, "diagnostics/sortedNodes");
Set<String> liveNodes = new HashSet<>();
SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) {
@Override
@ -283,7 +285,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
@Override
public ClusterStateProvider getClusterStateProvider() {
if (clusterState == null) {
Map map = (Map) Utils.getObjectByPath (m, false, "cluster/collections");
Map map = (Map) getObjectByPath(m, false, "cluster/collections");
if (map == null) map = new HashMap<>();
clusterState = ClusterState.load(0, map, liveNodes, "/clusterstate.json");
}
@ -311,7 +313,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
public void testHostAttribute() {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testHostAttribute.json");
Map<String, Object> conf = (Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config");
Map<String, Object> conf = (Map<String, Object>) getObjectByPath(m, false, "diagnostics/config");
Policy policy = new Policy(conf);
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
@ -331,7 +333,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testSysPropSuggestions.json");
Map<String, Object> conf = (Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config");
Map<String, Object> conf = (Map<String, Object>) getObjectByPath(m, false, "diagnostics/config");
Policy policy = new Policy(conf);
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
@ -349,7 +351,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
}
public void testSuggestionsRebalanceOnly() throws IOException {
public void testSuggestionsRebalanceOnly() {
String conf = " {" +
" 'cluster-preferences':[{" +
" 'minimize':'cores'," +
@ -370,11 +372,11 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
assertEquals("127.0.0.1:63219_solr", suggestions.get(1)._get("operation/command/move-replica/targetNode", null));
}
public void testSuggestionsRebalance2() throws IOException {
public void testSuggestionsRebalance2() {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testSuggestionsRebalance2.json");
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"));
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
assertEquals(3, suggestions.size());
@ -386,10 +388,10 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
public void testAddMissingReplica() throws IOException {
public void testAddMissingReplica() {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testAddMissingReplica.json");
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"));
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
@ -401,11 +403,10 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
public void testCreateCollectionWithEmptyPolicy() throws IOException {
public void testCreateCollectionWithEmptyPolicy() {
Map m = (Map) loadFromResource("testCreateCollectionWithEmptyPolicy.json");
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
AutoScalingConfig autoScalingConfig = new AutoScalingConfig(new HashMap());
///Users/noble/work/4solr/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java
//POSITIONS : [shard1:1[NRT] @127.0.0.1:49469_solr, shard1:2[NRT] @127.0.0.1:49469_solr]
List<ReplicaPosition> positions = PolicyHelper.getReplicaLocations("coll_new", autoScalingConfig, cloudManagerFromDiagnostics,
EMPTY_MAP, Collections.singletonList("shard1"), 2, 0, 0, null);
@ -417,6 +418,20 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
public void testUnresolvedSuggestion() {
Map<String, Object> m = (Map<String, Object>) loadFromResource("testUnresolvedSuggestion.json");
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"))
, cloudManagerFromDiagnostics);
// System.out.println(Utils.writeJson(suggestions, new StringWriter(), true).toString());
for (Suggester.SuggestionInfo suggestion : suggestions) {
assertEquals("unresolved-violation", suggestion._get("type", null));
assertEquals("1.0", suggestion._getStr("violation/violation/delta", null));
}
}
public static Object loadFromResource(String file) {
try (InputStream is = TestPolicy2.class.getResourceAsStream("/solrj/solr/autoscaling/" + file)) {
return Utils.fromJSON(is);

4
solr/webapp/web/js/angular/controllers/cluster-suggestions.js vendored
View File

@ -43,10 +43,10 @@ function($scope, $http, Constants) {
x.loading = false;
x.done = true;
x.run=true;
$scope.msg = "Post Data Submitted Successfully!";
$scope.msg = "Command Submitted Successfully!";
}, function (response) {
x.failed=true;
$scope.msg = "Service not Exists";
$scope.msg = "Service does not exist";
$scope.statusval = response.status;
$scope.statustext = response.statusText;
$scope.headers = response.headers();

10
solr/webapp/web/partials/cluster_suggestions.html
View File

@ -22,24 +22,24 @@
<tr>
<td>Type</td>
<td>Reason</td>
<td>Details</td>
<td>Action</td>
</tr>
<tr ng-if="parsedData.length==0">
<td>NA</td>
<td>NA</td>
<td>NA</td>
<td align="center" colspan="4">NA</td>
</tr>
<tr ng-repeat="x in parsedData">
<td>{{ x.type }}</td>
<td>{{ x.violation.clause }}</td>
<td>node: {{ x.violation.node }}, collection : {{ x.violation.collection }}, shard : {{ x.violation.shard }} &nbsp; </td>
<td>
<div class="s-container">
<input class="s-box1" ng-hide="x.run" type="button"
<input class="s-box1" ng-hide="x.run || x.type =='unresolved-violation'" type="button"
title="{{ x.operation }}" ng-click="postdata(x)" ng-mouseover="showPopover()"
ng-mouseleave="hidePopover()"/>
<div class="s-box2" ng-show="x.loading"><img src="img/loader.gif"></div>
<div class="s-box3" ng-show="x.done"><img src="img/ico/tick.png"></div>
<div class="s-box4" ng-show="x.failed"><img src="img/ico/cross.png"></div>
<div class="s-box4" ng-show="x.failed || x.type == 'unresolved-violation'"><img src="img/ico/cross.png"></div>
</div>
</td>
</tr>