Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11064: Collection APIs should use the disk space hint when using policy framework

This commit is contained in:
Noble Paul 2018-01-17 03:04:34 +11:00
parent 1c4b417c50
commit d99799c75c
5 changed files with 131 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -87,6 +87,8 @@ New Features
* SOLR-3218: Added range faceting support for CurrencyFieldType. This includes both "facet.range" as well
as json.facet's "type:range" (Andrew Morrison, Jan Høydahl, Vitaliy Zhovtyuk, hossman)
* SOLR-11064: Collection APIs should use the disk space hint when using policy framework (noble)
Bug Fixes
----------------------

7
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
View File

@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
/**
* Base class for overriding some behavior of {@link ClusterStateProvider}
@ -89,6 +90,12 @@ public class DelegatingClusterStateProvider implements ClusterStateProvider {
}
}
@Override
public DocCollection getCollection(String name) throws IOException {
ClusterState cs = getClusterState();
return cs == null ? null : cs.getCollectionOrNull(name);
}
@Override
public void connect() {
if (delegate != null) {

33
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
View File

@ -21,7 +21,9 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -32,6 +34,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.util.Pair;
@ -40,7 +43,10 @@ import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.util.Utils.time;
@ -106,6 +112,30 @@ public class PolicyHelper {
}
session = sessionWrapper.session;
Map<String, Double> diskSpaceReqd = new HashMap<>();
try {
DocCollection coll = cloudManager.getClusterStateProvider().getCollection(collName);
if (coll != null) {
for (String shardName : shardNames) {
Replica ldr = coll.getLeader(shardName);
if (ldr != null) {
Map<String, Map<String, List<ReplicaInfo>>> details = cloudManager.getNodeStateProvider().getReplicaInfo(ldr.getNodeName(),
Collections.singleton(FREEDISK.perReplicaValue));
ReplicaInfo replicaInfo = details.getOrDefault(collName, emptyMap()).getOrDefault(shardName, singletonList(null)).get(0);
if (replicaInfo != null) {
Object idxSz = replicaInfo.getVariables().get(FREEDISK.perReplicaValue);
if (idxSz != null) {
diskSpaceReqd.put(shardName, 1.5 * (Double) Suggestion.ConditionType.FREEDISK.validate(null, idxSz, false));
}
}
}
}
}
} catch (IOException e) {
/*ignore*/
}
Map<Replica.Type, Integer> typeVsCount = new EnumMap<>(Replica.Type.class);
typeVsCount.put(Replica.Type.NRT, nrtReplicas);
@ -123,6 +153,9 @@ public class PolicyHelper {
suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
}
}
if (diskSpaceReqd.get(shardName) != null) {
suggester.hint(Hint.MINFREEDISK, diskSpaceReqd.get(shardName));
}
SolrRequest op = suggester.getSuggestion();
if (op == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules " +

7
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
View File

@ -17,12 +17,13 @@
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.util.Map;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.solr.common.SolrCloseable;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
public interface ClusterStateProvider extends SolrCloseable {
@ -48,6 +49,10 @@ public interface ClusterStateProvider extends SolrCloseable {
*/
ClusterState getClusterState() throws IOException;
default DocCollection getCollection(String name) throws IOException{
return getClusterState().getCollectionOrNull(name);
}
/**
* Obtain cluster properties.
* @return configured cluster properties, or an empty map, never null.

83
solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
View File

@ -40,6 +40,8 @@ import org.apache.solr.client.solrj.cloud.DistributedQueueFactory;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.ZkStateReader;
@ -54,6 +56,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@ -1662,4 +1665,84 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("127.0.0.1:51147_solr" , op.getNode());
}
public void testDiskSpaceReqd() {
String autoScaleJson = "{" +
" cluster-preferences: [" +
" { minimize : cores, precision: 2}" +
" ]," +
" cluster-policy: [" +
" { replica : '0' , nodeRole: overseer}" +
" ]" +
"}";
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," +
"node2:{cores:4, freedisk: 262, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," +
"node3:{cores:7, freedisk: 749, heap:7834, sysprop.rack:rack4}," +
"node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" +
"}");
SolrCloudManager cloudManager = new DelegatingCloudManager(null) {
@Override
public NodeStateProvider getNodeStateProvider() {
return new DelegatingNodeStateProvider(null) {
@Override
public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
Map<String, Object> result = new LinkedHashMap<>();
keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
return result;
}
@Override
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
if (node.equals("node1")) {
Map m = Utils.makeMap("newColl",
Utils.makeMap("shard1", Collections.singletonList(new ReplicaInfo("r1", "shard1",
new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node1")),
Utils.makeMap(FREEDISK.perReplicaValue, 200)))));
return m;
} else if (node.equals("node2")) {
Map m = Utils.makeMap("newColl",
Utils.makeMap("shard2", Collections.singletonList(new ReplicaInfo("r1", "shard2",
new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node2")),
Utils.makeMap(FREEDISK.perReplicaValue, 200)))));
return m;
}
return Collections.emptyMap();
}
};
}
@Override
public ClusterStateProvider getClusterStateProvider() {
return new DelegatingClusterStateProvider(null) {
@Override
public Set<String> getLiveNodes() {
return new HashSet<>(Arrays.asList("node1", "node2", "node3", "node4"));
}
@Override
public DocCollection getCollection(String name) throws IOException {
return new DocCollection(name, Collections.emptyMap(), Collections.emptyMap(), DocRouter.DEFAULT) {
@Override
public Replica getLeader(String sliceName) {
if (sliceName.equals("shard1"))
return new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node1"));
if (sliceName.equals("shard2"))
return new Replica("r2", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node2"));
return null;
}
};
}
};
}
};
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson)),
cloudManager, null, Arrays.asList("shard1", "shard2"), 1, 0, 0, null);
assertTrue(locations.stream().allMatch(it -> "node3".equals(it.node)));
}
}