mirror of https://github.com/apache/lucene.git
SOLR-5132: Added a new collection action MODIFYCOLLECTION
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1680052 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul
parent
3827a5fb10
commit
aecf45429c
|
|
@ -569,12 +569,14 @@ New Features
|
|||
* SOLR-7226: Make /query/* jmx/* , requestDispatcher/*, <listener> <initParams>
|
||||
properties in solrconfig.xml editable (Noble Paul)
|
||||
|
||||
* SOLR-7240: '/' redirects to '/solr/' for convinience (Martijn Koster, hossman)
|
||||
* SOLR-7240: '/' redirects to '/solr/' for convenience (Martijn Koster, hossman)
|
||||
|
||||
* SOLR-5911: Added payload support for term vectors. New "termPayloads" option for fields
|
||||
/ types in the schema, and "tv.payloads" param for the term vector component.
|
||||
(Mike McCandless, David Smiley)
|
||||
|
||||
* SOLR-5132: Added a new collection action MODIFYCOLLECTION (Noble Paul)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
|
|||
import org.apache.solr.common.params.CollectionParams;
|
||||
import org.apache.solr.common.util.IOUtils;
|
||||
import org.apache.solr.core.CloudConfig;
|
||||
import org.apache.solr.handler.admin.CollectionsHandler;
|
||||
import org.apache.solr.handler.component.ShardHandler;
|
||||
import org.apache.solr.update.UpdateShardHandler;
|
||||
import org.apache.solr.util.stats.Clock;
|
||||
|
|
@ -353,6 +354,9 @@ public class Overseer implements Closeable {
|
|||
return new ZkWriteCommand(collName, dProp.getDocCollection());
|
||||
}
|
||||
break;
|
||||
case MODIFYCOLLECTION:
|
||||
CollectionsHandler.verifyRuleParams(zkController.getCoreContainer() ,message.getProperties());
|
||||
return new CollectionMutator(reader).modifyCollection(clusterState,message);
|
||||
default:
|
||||
throw new RuntimeException("unknown operation:" + operation
|
||||
+ " contents:" + message.getProperties());
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ package org.apache.solr.cloud;
|
|||
*/
|
||||
|
||||
import static org.apache.solr.cloud.Assign.*;
|
||||
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.*;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
|
||||
import static org.apache.solr.common.params.CommonParams.*;
|
||||
|
|
@ -135,13 +136,13 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
public int maxParallelThreads = 10;
|
||||
|
||||
public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
|
||||
public static final Map<String, Object> COLL_PROPS = Collections.unmodifiableMap(makeMap(
|
||||
ROUTER, DocRouter.DEFAULT_NAME,
|
||||
ZkStateReader.REPLICATION_FACTOR, "1",
|
||||
ZkStateReader.MAX_SHARDS_PER_NODE, "1",
|
||||
ZkStateReader.AUTO_ADD_REPLICAS, "false",
|
||||
"rule", null,
|
||||
"snitch",null);
|
||||
DocCollection.RULE, null,
|
||||
DocCollection.SNITCH, null));
|
||||
|
||||
static final Random RANDOM;
|
||||
static {
|
||||
|
|
@ -615,6 +616,9 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
case REBALANCELEADERS:
|
||||
processRebalanceLeaders(message);
|
||||
break;
|
||||
case MODIFYCOLLECTION:
|
||||
overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message));
|
||||
break;
|
||||
default:
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
|
||||
+ operation);
|
||||
|
|
@ -1045,7 +1049,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
|
||||
|
||||
// assume the core exists and try to unload it
|
||||
Map m = ZkNodeProps.makeMap("qt", adminPath, CoreAdminParams.ACTION,
|
||||
Map m = makeMap("qt", adminPath, CoreAdminParams.ACTION,
|
||||
CoreAdminAction.UNLOAD.toString(), CoreAdminParams.CORE, core,
|
||||
CoreAdminParams.DELETE_INSTANCE_DIR, "true",
|
||||
CoreAdminParams.DELETE_DATA_DIR, "true");
|
||||
|
|
@ -1292,8 +1296,8 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
|
||||
DocCollection collection = clusterState.getCollection(collectionName);
|
||||
int maxShardsPerNode = collection.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1);
|
||||
int repFactor = message.getInt(ZkStateReader.REPLICATION_FACTOR, collection.getInt(ZkStateReader.REPLICATION_FACTOR, 1));
|
||||
int maxShardsPerNode = collection.getInt(MAX_SHARDS_PER_NODE, 1);
|
||||
int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1));
|
||||
String createNodeSetStr = message.getStr(CREATE_NODE_SET);
|
||||
|
||||
ArrayList<Node> sortedNodeList = getNodesForNewShard(clusterState, collectionName, numSlices, maxShardsPerNode, repFactor, createNodeSetStr);
|
||||
|
|
@ -1981,7 +1985,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName();
|
||||
if (clusterState.hasCollection(tempSourceCollectionName)) {
|
||||
log.info("Deleting temporary collection: " + tempSourceCollectionName);
|
||||
Map<String, Object> props = ZkNodeProps.makeMap(
|
||||
Map<String, Object> props = makeMap(
|
||||
Overseer.QUEUE_OPERATION, DELETE.toLower(),
|
||||
NAME, tempSourceCollectionName);
|
||||
|
||||
|
|
@ -2065,10 +2069,10 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
// create a temporary collection with just one node on the shard leader
|
||||
String configName = zkStateReader.readConfigName(sourceCollection.getName());
|
||||
Map<String, Object> props = ZkNodeProps.makeMap(
|
||||
Map<String, Object> props = makeMap(
|
||||
Overseer.QUEUE_OPERATION, CREATE.toLower(),
|
||||
NAME, tempSourceCollectionName,
|
||||
ZkStateReader.REPLICATION_FACTOR, 1,
|
||||
REPLICATION_FACTOR, 1,
|
||||
NUM_SLICES, 1,
|
||||
COLL_CONF, configName,
|
||||
CREATE_NODE_SET, sourceLeader.getNodeName());
|
||||
|
|
@ -2194,7 +2198,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
try {
|
||||
log.info("Deleting temporary collection: " + tempSourceCollectionName);
|
||||
props = ZkNodeProps.makeMap(
|
||||
props = makeMap(
|
||||
Overseer.QUEUE_OPERATION, DELETE.toLower(),
|
||||
NAME, tempSourceCollectionName);
|
||||
deleteCollection(new ZkNodeProps(props), results);
|
||||
|
|
@ -2293,7 +2297,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
// look at the replication factor and see if it matches reality
|
||||
// if it does not, find best nodes to create more cores
|
||||
|
||||
int repFactor = message.getInt(ZkStateReader.REPLICATION_FACTOR, 1);
|
||||
int repFactor = message.getInt(REPLICATION_FACTOR, 1);
|
||||
|
||||
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
|
||||
String async = null;
|
||||
|
|
@ -2312,10 +2316,10 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
ClusterStateMutator.getShardNames(numSlices, shardNames);
|
||||
}
|
||||
|
||||
int maxShardsPerNode = message.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1);
|
||||
int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
|
||||
|
||||
if (repFactor <= 0) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.REPLICATION_FACTOR + " must be greater than 0");
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, REPLICATION_FACTOR + " must be greater than 0");
|
||||
}
|
||||
|
||||
if (numSlices <= 0) {
|
||||
|
|
@ -2330,7 +2334,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
if (repFactor > nodeList.size()) {
|
||||
log.warn("Specified "
|
||||
+ ZkStateReader.REPLICATION_FACTOR
|
||||
+ REPLICATION_FACTOR
|
||||
+ " of "
|
||||
+ repFactor
|
||||
+ " on collection "
|
||||
|
|
@ -2344,11 +2348,11 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
int requestedShardsToCreate = numSlices * repFactor;
|
||||
if (maxShardsAllowedToCreate < requestedShardsToCreate) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
|
||||
+ ZkStateReader.MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
|
||||
+ MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
|
||||
+ ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size()
|
||||
+ ". This allows a maximum of " + maxShardsAllowedToCreate
|
||||
+ " to be created. Value of " + NUM_SLICES + " is " + numSlices
|
||||
+ " and value of " + ZkStateReader.REPLICATION_FACTOR + " is " + repFactor
|
||||
+ " and value of " + REPLICATION_FACTOR + " is " + repFactor
|
||||
+ ". This requires " + requestedShardsToCreate
|
||||
+ " shards to be created (higher than the allowed number)");
|
||||
}
|
||||
|
|
@ -2556,7 +2560,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
|
||||
|
||||
if (node == null) {
|
||||
node = getNodesForNewShard(clusterState, collection, coll.getSlices().size(), coll.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1), coll.getInt(ZkStateReader.REPLICATION_FACTOR, 1), null).get(0).nodeName;
|
||||
node = getNodesForNewShard(clusterState, collection, coll.getSlices().size(), coll.getInt(MAX_SHARDS_PER_NODE, 1), coll.getInt(REPLICATION_FACTOR, 1), null).get(0).nodeName;
|
||||
log.info("Node not provided, Identified {} for creating new replica", node);
|
||||
}
|
||||
|
||||
|
|
@ -2684,7 +2688,7 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
if (configName != null) {
|
||||
String collDir = ZkStateReader.COLLECTIONS_ZKNODE + "/" + coll;
|
||||
log.info("creating collections conf node {} ", collDir);
|
||||
byte[] data = ZkStateReader.toJSON(ZkNodeProps.makeMap(ZkController.CONFIGNAME_PROP, configName));
|
||||
byte[] data = ZkStateReader.toJSON(makeMap(ZkController.CONFIGNAME_PROP, configName));
|
||||
if (zkStateReader.getZkClient().exists(collDir, true)) {
|
||||
zkStateReader.getZkClient().setData(collDir, data, true);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -29,9 +29,11 @@ import org.apache.solr.common.cloud.Replica;
|
|||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.cloud.ZkNodeProps;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.handler.admin.CollectionsHandler;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||
import static org.apache.solr.common.params.CommonParams.NAME;
|
||||
|
||||
public class CollectionMutator {
|
||||
|
|
@ -84,6 +86,17 @@ public class CollectionMutator {
|
|||
return new ZkWriteCommand(collection, newCollection);
|
||||
}
|
||||
|
||||
public ZkWriteCommand modifyCollection(final ClusterState clusterState, ZkNodeProps message){
|
||||
if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
|
||||
DocCollection coll = clusterState.getCollection(message.getStr(COLLECTION_PROP));
|
||||
Map<String, Object> m = coll.shallowCopy();
|
||||
for (String prop : CollectionsHandler.MODIFIABLE_COLL_PROPS) {
|
||||
if(message.get(prop)!= null) m.put(prop,message.get(prop));
|
||||
}
|
||||
return new ZkWriteCommand(coll.getName(),
|
||||
new DocCollection(coll.getName(),coll.getSlicesMap(),m,coll.getRouter(),coll.getZNodeVersion(),coll.getZNode()));
|
||||
}
|
||||
|
||||
public static DocCollection updateSlice(String collectionName, DocCollection collection, Slice slice) {
|
||||
DocCollection newCollection = null;
|
||||
Map<String, Slice> slices;
|
||||
|
|
|
|||
|
|
@ -326,62 +326,39 @@ public class ReplicaAssigner {
|
|||
|
||||
public Map<String, SnitchContext> failedNodes = new HashMap<>();
|
||||
|
||||
static class SnitchInfoImpl extends SnitchContext.SnitchInfo {
|
||||
final Snitch snitch;
|
||||
final Set<String> myTags = new HashSet<>();
|
||||
final Map<String, SnitchContext> nodeVsContext = new HashMap<>();
|
||||
private final CoreContainer cc;
|
||||
|
||||
SnitchInfoImpl(Map<String, Object> conf, Snitch snitch, CoreContainer cc) {
|
||||
super(conf);
|
||||
this.snitch = snitch;
|
||||
this.cc = cc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getTagNames() {
|
||||
return myTags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CoreContainer getCoreContainer() {
|
||||
return cc;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method uses the snitches and get the tags for all the nodes
|
||||
*/
|
||||
private Map<String, Map<String, Object>> getTagsForNodes(final CoreContainer cc, List snitchConf) {
|
||||
|
||||
class Info extends SnitchContext.SnitchInfo {
|
||||
final Snitch snitch;
|
||||
final Set<String> myTags = new HashSet<>();
|
||||
final Map<String, SnitchContext> nodeVsContext = new HashMap<>();
|
||||
|
||||
Info(Map<String, Object> conf, Snitch snitch) {
|
||||
super(conf);
|
||||
this.snitch = snitch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getTagNames() {
|
||||
return myTags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CoreContainer getCoreContainer() {
|
||||
return cc;
|
||||
}
|
||||
}
|
||||
|
||||
Map<Class, Info> snitches = new LinkedHashMap<>();
|
||||
for (Object o : snitchConf) {
|
||||
//instantiating explicitly specified snitches
|
||||
String klas = null;
|
||||
Map map = Collections.emptyMap();
|
||||
if (o instanceof Map) {//it can be a Map
|
||||
map = (Map) o;
|
||||
klas = (String) map.get("class");
|
||||
if (klas == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "snitch must have a class attribute");
|
||||
}
|
||||
} else { //or just the snitch name
|
||||
klas = o.toString();
|
||||
}
|
||||
try {
|
||||
if (klas.indexOf('.') == -1) klas = Snitch.class.getPackage().getName() + "." + klas;
|
||||
Snitch inst = cc == null ?
|
||||
(Snitch) Snitch.class.getClassLoader().loadClass(klas).newInstance() :
|
||||
cc.getResourceLoader().newInstance(klas, Snitch.class);
|
||||
snitches.put(inst.getClass(), new Info(map, inst));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Map<Class, SnitchInfoImpl> snitches = getSnitchInfos(cc, snitchConf);
|
||||
for (Class c : Snitch.WELL_KNOWN_SNITCHES) {
|
||||
if (snitches.containsKey(c)) continue;// it is already specified explicitly , ignore
|
||||
try {
|
||||
snitches.put(c, new Info(Collections.EMPTY_MAP, (Snitch) c.newInstance()));
|
||||
snitches.put(c, new SnitchInfoImpl(Collections.EMPTY_MAP, (Snitch) c.newInstance(), cc));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error instantiating Snitch " + c.getName());
|
||||
}
|
||||
|
|
@ -389,7 +366,7 @@ public class ReplicaAssigner {
|
|||
for (String tagName : tagNames) {
|
||||
//identify which snitch is going to provide values for a given tag
|
||||
boolean foundProvider = false;
|
||||
for (Info info : snitches.values()) {
|
||||
for (SnitchInfoImpl info : snitches.values()) {
|
||||
if (info.snitch.isKnownTag(tagName)) {
|
||||
foundProvider = true;
|
||||
info.myTags.add(tagName);
|
||||
|
|
@ -403,7 +380,7 @@ public class ReplicaAssigner {
|
|||
|
||||
for (String node : liveNodes) {
|
||||
//now use the Snitch to get the tags
|
||||
for (Info info : snitches.values()) {
|
||||
for (SnitchInfoImpl info : snitches.values()) {
|
||||
if (!info.myTags.isEmpty()) {
|
||||
SnitchContext context = new SnitchContext(info, node);
|
||||
info.nodeVsContext.put(node, context);
|
||||
|
|
@ -417,7 +394,7 @@ public class ReplicaAssigner {
|
|||
}
|
||||
|
||||
Map<String, Map<String, Object>> result = new HashMap<>();
|
||||
for (Info info : snitches.values()) {
|
||||
for (SnitchInfoImpl info : snitches.values()) {
|
||||
for (Map.Entry<String, SnitchContext> e : info.nodeVsContext.entrySet()) {
|
||||
SnitchContext context = e.getValue();
|
||||
String node = e.getKey();
|
||||
|
|
@ -448,4 +425,40 @@ public class ReplicaAssigner {
|
|||
|
||||
}
|
||||
|
||||
public static void verifySnitchConf(CoreContainer cc, List snitchConf) {
|
||||
getSnitchInfos(cc, snitchConf);
|
||||
}
|
||||
|
||||
|
||||
static Map<Class, SnitchInfoImpl> getSnitchInfos(CoreContainer cc, List snitchConf) {
|
||||
Map<Class, SnitchInfoImpl> snitches = new LinkedHashMap<>();
|
||||
if (snitchConf == null) return snitches;
|
||||
for (Object o : snitchConf) {
|
||||
//instantiating explicitly specified snitches
|
||||
String klas = null;
|
||||
Map map = Collections.emptyMap();
|
||||
if (o instanceof Map) {//it can be a Map
|
||||
map = (Map) o;
|
||||
klas = (String) map.get("class");
|
||||
if (klas == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "snitch must have a class attribute");
|
||||
}
|
||||
} else { //or just the snitch name
|
||||
klas = o.toString();
|
||||
}
|
||||
try {
|
||||
if (klas.indexOf('.') == -1) klas = Snitch.class.getPackage().getName() + "." + klas;
|
||||
Snitch inst = cc == null ?
|
||||
(Snitch) Snitch.class.getClassLoader().loadClass(klas).newInstance() :
|
||||
cc.getResourceLoader().newInstance(klas, Snitch.class);
|
||||
snitches.put(inst.getClass(), new SnitchInfoImpl(map, inst, cc));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return snitches;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
|
|||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -27,6 +28,7 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
|
|
@ -35,12 +37,15 @@ import org.apache.solr.client.solrj.request.CoreAdminRequest;
|
|||
import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard;
|
||||
import org.apache.solr.cloud.DistributedQueue;
|
||||
import org.apache.solr.cloud.DistributedQueue.QueueEvent;
|
||||
import org.apache.solr.cloud.Overseer;
|
||||
import org.apache.solr.cloud.OverseerSolrResponse;
|
||||
import org.apache.solr.cloud.overseer.SliceMutator;
|
||||
import org.apache.solr.cloud.rule.ReplicaAssigner;
|
||||
import org.apache.solr.cloud.rule.Rule;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.ImplicitDocRouter;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.cloud.ZkCmdExecutor;
|
||||
|
|
@ -76,6 +81,8 @@ import static org.apache.solr.cloud.OverseerCollectionProcessor.REQUESTID;
|
|||
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARD_UNIQUE;
|
||||
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
|
||||
import static org.apache.solr.common.cloud.DocCollection.RULE;
|
||||
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
|
||||
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||
|
|
@ -91,6 +98,7 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
|
|||
import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR;
|
||||
import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR;
|
||||
import static org.apache.solr.common.params.ShardParams._ROUTE_;
|
||||
import static org.apache.solr.common.util.StrUtils.formatString;
|
||||
|
||||
public class CollectionsHandler extends RequestHandlerBase {
|
||||
protected static Logger log = LoggerFactory.getLogger(CollectionsHandler.class);
|
||||
|
|
@ -157,7 +165,9 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
if (result != null) {
|
||||
result.put(QUEUE_OPERATION, operation.action.toLower());
|
||||
ZkNodeProps props = new ZkNodeProps(result);
|
||||
handleResponse(operation.action.toLower(), props, rsp, operation.timeOut);
|
||||
if (operation.sendToOCPQueue) handleResponse(operation.action.toLower(), props, rsp, operation.timeOut);
|
||||
else Overseer.getInQueue(coreContainer.getZkController().getZkClient()).offer(ZkStateReader.toJSON(props));
|
||||
|
||||
}
|
||||
} else {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
|
||||
|
|
@ -287,7 +297,6 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
Map<String, Object> props = req.getParams().required().getAll(null, NAME);
|
||||
props.put("fromApi", "true");
|
||||
req.getParams().getAll(props,
|
||||
NAME,
|
||||
REPLICATION_FACTOR,
|
||||
COLL_CONF,
|
||||
NUM_SLICES,
|
||||
|
|
@ -296,16 +305,18 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
SHARDS_PROP,
|
||||
ASYNC,
|
||||
STATE_FORMAT,
|
||||
AUTO_ADD_REPLICAS);
|
||||
AUTO_ADD_REPLICAS,
|
||||
RULE,
|
||||
SNITCH);
|
||||
|
||||
if (props.get(STATE_FORMAT) == null) {
|
||||
props.put(STATE_FORMAT, "2");
|
||||
}
|
||||
addRuleMap(req.getParams(), props, "rule");
|
||||
addRuleMap(req.getParams(), props, "snitch");
|
||||
|
||||
addMapObject(props, RULE);
|
||||
addMapObject(props, SNITCH);
|
||||
verifyRuleParams(h.coreContainer, props);
|
||||
if (SYSTEM_COLL.equals(props.get(NAME))) {
|
||||
//We must always create asystem collection with only a single shard
|
||||
//We must always create a .system collection with only a single shard
|
||||
props.put(NUM_SLICES, 1);
|
||||
props.remove(SHARDS_PROP);
|
||||
createSysConfigSet(h.coreContainer);
|
||||
|
|
@ -316,15 +327,6 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
|
||||
}
|
||||
|
||||
private void addRuleMap(SolrParams params, Map<String, Object> props, String key) {
|
||||
String[] rules = params.getParams(key);
|
||||
if (rules != null && rules.length > 0) {
|
||||
ArrayList<Map> l = new ArrayList<>();
|
||||
for (String rule : rules) l.add(Rule.parseRule(rule));
|
||||
props.put(key, l);
|
||||
}
|
||||
}
|
||||
|
||||
private void createSysConfigSet(CoreContainer coreContainer) throws KeeperException, InterruptedException {
|
||||
SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient();
|
||||
ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zk.getZkClientTimeout());
|
||||
|
|
@ -396,7 +398,7 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
}
|
||||
|
||||
},
|
||||
SPLITSHARD_OP(SPLITSHARD, DEFAULT_ZK_TIMEOUT * 5) {
|
||||
SPLITSHARD_OP(SPLITSHARD, DEFAULT_ZK_TIMEOUT * 5, true) {
|
||||
@Override
|
||||
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h)
|
||||
throws Exception {
|
||||
|
|
@ -670,17 +672,35 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
new RebalanceLeaders(req,rsp,h).execute();
|
||||
return null;
|
||||
}
|
||||
},
|
||||
MODIFYCOLLECTION_OP(MODIFYCOLLECTION, DEFAULT_ZK_TIMEOUT, false) {
|
||||
@Override
|
||||
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
|
||||
|
||||
Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS.toArray(new String[0]));
|
||||
if (m.isEmpty()) throw new SolrException(ErrorCode.BAD_REQUEST,
|
||||
formatString("no supported values provided rule, snitch, masShardsPerNode, replicationFactor"));
|
||||
req.getParams().required().getAll(m, COLLECTION_PROP);
|
||||
addMapObject(m, RULE);
|
||||
addMapObject(m, SNITCH);
|
||||
for (String prop : MODIFIABLE_COLL_PROPS) DocCollection.verifyProp(m, prop);
|
||||
verifyRuleParams(h.coreContainer, m);
|
||||
return m;
|
||||
}
|
||||
};
|
||||
CollectionAction action;
|
||||
long timeOut;
|
||||
boolean sendToOCPQueue;
|
||||
|
||||
CollectionOperation(CollectionAction action) {
|
||||
this(action, DEFAULT_ZK_TIMEOUT);
|
||||
this(action, DEFAULT_ZK_TIMEOUT, true);
|
||||
}
|
||||
|
||||
CollectionOperation(CollectionAction action, long timeOut) {
|
||||
CollectionOperation(CollectionAction action, long timeOut, boolean sendToOCPQueue) {
|
||||
this.action = action;
|
||||
this.timeOut = timeOut;
|
||||
this.sendToOCPQueue = sendToOCPQueue;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -697,4 +717,46 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
}
|
||||
}
|
||||
|
||||
public static void verifyRuleParams(CoreContainer cc, Map<String, Object> m) {
|
||||
List l = (List) m.get(RULE);
|
||||
if (l != null) {
|
||||
for (Object o : l) {
|
||||
Map map = (Map) o;
|
||||
try {
|
||||
new Rule(map);
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error in rule " + m, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
ReplicaAssigner.verifySnitchConf(cc, (List) m.get(SNITCH));
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a String of the form a:b,c:d to a Map
|
||||
*/
|
||||
private static Map<String, Object> addMapObject(Map<String, Object> props, String key) {
|
||||
Object v = props.get(key);
|
||||
if (v == null) return props;
|
||||
List<String> val = new ArrayList<>();
|
||||
if (v instanceof String[]) {
|
||||
val.addAll(Arrays.asList((String[]) v));
|
||||
} else {
|
||||
val.add(v.toString());
|
||||
}
|
||||
if (val.size() > 0) {
|
||||
ArrayList<Map> l = new ArrayList<>();
|
||||
for (String rule : val) l.add(Rule.parseRule(rule));
|
||||
props.put(key, l);
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
public static final List<String> MODIFIABLE_COLL_PROPS = ImmutableList.of(
|
||||
RULE,
|
||||
SNITCH,
|
||||
REPLICATION_FACTOR,
|
||||
MAX_SHARDS_PER_NODE,
|
||||
AUTO_ADD_REPLICAS);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,15 +21,24 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.request.GenericSolrRequest;
|
||||
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.cloud.OverseerCollectionProcessor;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
|
||||
import static org.apache.solr.core.CoreContainer.COLLECTIONS_HANDLER_PATH;
|
||||
|
||||
public class RulesTest extends AbstractFullDistribZkTestBase {
|
||||
static final Logger log = LoggerFactory.getLogger(RulesTest.class);
|
||||
|
||||
|
|
@ -62,6 +71,53 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testModifyColl() throws Exception {
|
||||
String rulesColl = "modifyColl";
|
||||
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
|
||||
CollectionAdminResponse rsp;
|
||||
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
|
||||
create.setCollectionName(rulesColl);
|
||||
create.setNumShards(1);
|
||||
create.setReplicationFactor(2);
|
||||
create.setRule("cores:<4", "node:*,replica:1", "freedisk:>1");
|
||||
create.setSnitch("class:ImplicitSnitch");
|
||||
rsp = create.process(client);
|
||||
assertEquals(0, rsp.getStatus());
|
||||
assertTrue(rsp.isSuccess());
|
||||
ModifiableSolrParams p = new ModifiableSolrParams();
|
||||
p.add("collection", rulesColl);
|
||||
p.add("action", "MODIFYCOLLECTION");
|
||||
p.add("rule", "cores:<5");
|
||||
p.add("rule", "node:*,replica:1");
|
||||
p.add("rule", "freedisk:>5");
|
||||
p.add("autoAddReplicas", "true");
|
||||
client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
|
||||
}
|
||||
|
||||
|
||||
for (int i = 0; i < 20; i++) {
|
||||
DocCollection rulesCollection = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), rulesColl);
|
||||
log.info("version_of_coll {} ", rulesCollection.getZNodeVersion());
|
||||
List list = (List) rulesCollection.get("rule");
|
||||
assertEquals(3, list.size());
|
||||
if (!"<5".equals(((Map) list.get(0)).get("cores"))) {
|
||||
if (i < 19) {
|
||||
Thread.sleep(100);
|
||||
continue;
|
||||
}
|
||||
|
||||
}
|
||||
assertEquals("<5", ((Map) list.get(0)).get("cores"));
|
||||
assertEquals("1", ((Map) list.get(1)).get("replica"));
|
||||
assertEquals(">5", ((Map) list.get(2)).get("freedisk"));
|
||||
assertEquals("true", String.valueOf(rulesCollection.getProperties().get("autoAddReplicas")));
|
||||
list = (List) rulesCollection.get("snitch");
|
||||
assertEquals(1, list.size());
|
||||
assertEquals("ImplicitSnitch", ((Map) list.get(0)).get("class"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import java.util.Collection;
|
|||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
|
@ -28,6 +29,10 @@ import org.apache.solr.common.SolrException.ErrorCode;
|
|||
import org.noggit.JSONUtil;
|
||||
import org.noggit.JSONWriter;
|
||||
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
|
||||
|
||||
/**
|
||||
* Models a Collection in zookeeper (but that Java name is obviously taken, hence "DocCollection")
|
||||
*/
|
||||
|
|
@ -35,6 +40,8 @@ public class DocCollection extends ZkNodeProps {
|
|||
public static final String DOC_ROUTER = "router";
|
||||
public static final String SHARDS = "shards";
|
||||
public static final String STATE_FORMAT = "stateFormat";
|
||||
public static final String RULE = "rule";
|
||||
public static final String SNITCH = "snitch";
|
||||
private int znodeVersion = -1; // sentinel
|
||||
|
||||
private final String name;
|
||||
|
|
@ -45,7 +52,7 @@ public class DocCollection extends ZkNodeProps {
|
|||
|
||||
private final Integer replicationFactor;
|
||||
private final Integer maxShardsPerNode;
|
||||
private final boolean autoAddReplicas;
|
||||
private final Boolean autoAddReplicas;
|
||||
|
||||
|
||||
public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router) {
|
||||
|
|
@ -64,25 +71,12 @@ public class DocCollection extends ZkNodeProps {
|
|||
|
||||
this.slices = slices;
|
||||
this.activeSlices = new HashMap<>();
|
||||
Object replicationFactorObject = (Object) props.get(ZkStateReader.REPLICATION_FACTOR);
|
||||
if (replicationFactorObject != null) {
|
||||
this.replicationFactor = Integer.parseInt(replicationFactorObject.toString());
|
||||
} else {
|
||||
this.replicationFactor = null;
|
||||
}
|
||||
Object maxShardsPerNodeObject = (Object) props.get(ZkStateReader.MAX_SHARDS_PER_NODE);
|
||||
if (maxShardsPerNodeObject != null) {
|
||||
this.maxShardsPerNode = Integer.parseInt(maxShardsPerNodeObject.toString());
|
||||
} else {
|
||||
this.maxShardsPerNode = null;
|
||||
}
|
||||
Object autoAddReplicasObject = (Object) props.get(ZkStateReader.AUTO_ADD_REPLICAS);
|
||||
if (autoAddReplicasObject != null) {
|
||||
this.autoAddReplicas = Boolean.parseBoolean(autoAddReplicasObject.toString());
|
||||
} else {
|
||||
this.autoAddReplicas = false;
|
||||
}
|
||||
|
||||
this.replicationFactor = (Integer) verifyProp(props, REPLICATION_FACTOR);
|
||||
this.maxShardsPerNode = (Integer) verifyProp(props, MAX_SHARDS_PER_NODE);
|
||||
Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS);
|
||||
this.autoAddReplicas = autoAddReplicas == null ? false : autoAddReplicas;
|
||||
verifyProp(props, RULE);
|
||||
verifyProp(props, SNITCH);
|
||||
Iterator<Map.Entry<String, Slice>> iter = slices.entrySet().iterator();
|
||||
|
||||
while (iter.hasNext()) {
|
||||
|
|
@ -95,6 +89,24 @@ public class DocCollection extends ZkNodeProps {
|
|||
assert name != null && slices != null;
|
||||
}
|
||||
|
||||
public static Object verifyProp(Map<String, Object> props, String propName) {
|
||||
Object o = props.get(propName);
|
||||
if (o == null) return null;
|
||||
switch (propName) {
|
||||
case MAX_SHARDS_PER_NODE:
|
||||
case REPLICATION_FACTOR:
|
||||
return Integer.parseInt(o.toString());
|
||||
case AUTO_ADD_REPLICAS:
|
||||
return Boolean.parseBoolean(o.toString());
|
||||
case "snitch":
|
||||
case "rule":
|
||||
return (List) o;
|
||||
default:
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Unknown property " + propName);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**Use this to make an exact copy of DocCollection with a new set of Slices and every other property as is
|
||||
* @param slices the new set of Slices
|
||||
* @return the resulting DocCollection
|
||||
|
|
@ -164,7 +176,7 @@ public class DocCollection extends ZkNodeProps {
|
|||
|
||||
public int getMaxShardsPerNode() {
|
||||
if (maxShardsPerNode == null) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.MAX_SHARDS_PER_NODE + " is not in the cluster state.");
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, MAX_SHARDS_PER_NODE + " is not in the cluster state.");
|
||||
}
|
||||
return maxShardsPerNode;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ package org.apache.solr.common.params;
|
|||
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
public interface CollectionParams
|
||||
{
|
||||
/** What action **/
|
||||
|
|
@ -50,17 +52,17 @@ public interface CollectionParams
|
|||
ADDREPLICAPROP,
|
||||
DELETEREPLICAPROP,
|
||||
BALANCESHARDUNIQUE,
|
||||
REBALANCELEADERS;
|
||||
|
||||
public static CollectionAction get( String p )
|
||||
{
|
||||
REBALANCELEADERS,
|
||||
MODIFYCOLLECTION;
|
||||
|
||||
public static CollectionAction get(String p) {
|
||||
if( p != null ) {
|
||||
try {
|
||||
return CollectionAction.valueOf( p.toUpperCase(Locale.ROOT) );
|
||||
}
|
||||
catch( Exception ex ) {}
|
||||
}
|
||||
return null;
|
||||
return null;
|
||||
}
|
||||
public boolean isEqual(String s){
|
||||
if(s == null) return false;
|
||||
|
|
|
|||
Loading…
Reference in New Issue