Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9216: Support collection.configName in MODIFYCOLLECTION request

This commit is contained in:
Noble Paul 2016-06-30 12:10:36 +05:30
parent d1a9daec87
commit ce3ea76781
8 changed files with 177 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -40,6 +40,9 @@ New Features
* SOLR-7374: Core level Backup/Restore now supports specifying the directory implementation to use
via the "repository" parameter. (Hrishikesh Gadre, Varun Thacker, Mark Miller)
* SOLR-9216: Support collection.configName in MODIFYCOLLECTION request (Keith Laban, noble)
Bug Fixes
----------------------

63
solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
View File

@ -136,7 +136,7 @@ import static org.apache.solr.common.util.Utils.makeMap;
public class OverseerCollectionMessageHandler implements OverseerMessageHandler {
public static final String NUM_SLICES = "numShards";
static final boolean CREATE_NODE_SET_SHUFFLE_DEFAULT = true;
public static final String CREATE_NODE_SET_SHUFFLE = "createNodeSet.shuffle";
public static final String CREATE_NODE_SET_EMPTY = "EMPTY";
@ -275,7 +275,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
processRebalanceLeaders(message);
break;
case MODIFYCOLLECTION:
overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
modifyCollection(message, results);
break;
case MIGRATESTATEFORMAT:
migrateStateFormat(message, results);
@ -1824,6 +1824,25 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
return nodeList;
}
private void modifyCollection(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
//the rest of the processing is based on writing cluster state properties
//remove the property here to avoid any errors down the pipeline due to this property appearing
String configName = (String) message.getProperties().remove(COLL_CONF);
if(configName != null) {
validateConfigOrThrowSolrException(configName);
boolean isLegacyCloud = Overseer.isLegacy(zkStateReader);
createConfNode(configName, collectionName, isLegacyCloud);
reloadCollection(new ZkNodeProps(NAME, collectionName), results);
}
overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
}
private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
final String collectionName = message.getStr(NAME);
@ -1835,9 +1854,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
String configName = getConfigName(collectionName, message);
if (configName == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");
} else if (!validateConfig(configName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName);
}
validateConfigOrThrowSolrException(configName);
try {
// look at the replication factor and see if it matches reality
@ -2487,8 +2507,11 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
return configName;
}
private boolean validateConfig(String configName) throws KeeperException, InterruptedException {
return zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);
private void validateConfigOrThrowSolrException(String configName) throws KeeperException, InterruptedException {
boolean isValid = zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);
if(!isValid) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName);
}
}
/**
@ -2680,34 +2703,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
/* @Override
public void markExclusiveTask(String collectionName, ZkNodeProps message) {
if (collectionName != null) {
synchronized (collectionWip) {
collectionWip.add(collectionName);
}
}
}
@Override
public void unmarkExclusiveTask(String collectionName, String operation, ZkNodeProps message) {
if(collectionName != null) {
synchronized (collectionWip) {
collectionWip.remove(collectionName);
}
}
}*/
/*
@Override
public ExclusiveMarking checkExclusiveMarking(String collectionName, ZkNodeProps message) {
synchronized (collectionWip) {
if(collectionWip.contains(collectionName))
return ExclusiveMarking.NONEXCLUSIVE;
}
return ExclusiveMarking.NOTDETERMINED;
}*/
private long sessionId = -1;
private LockTree.Session lockSession;

11
solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
View File

@ -92,9 +92,18 @@ public class CollectionMutator {
if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
DocCollection coll = clusterState.getCollection(message.getStr(COLLECTION_PROP));
Map<String, Object> m = coll.shallowCopy();
boolean hasAnyOps = false;
for (String prop : CollectionsHandler.MODIFIABLE_COLL_PROPS) {
if(message.get(prop)!= null) m.put(prop,message.get(prop));
if(message.get(prop)!= null) {
hasAnyOps = true;
m.put(prop,message.get(prop));
}
}
if(!hasAnyOps) {
return ZkStateWriter.NO_OP;
}
return new ZkWriteCommand(coll.getName(),
new DocCollection(coll.getName(),coll.getSlicesMap(),m,coll.getRouter(),coll.getZNodeVersion(),coll.getZNode()));
}

25
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@ -51,8 +50,18 @@ import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.*;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@ -765,13 +774,13 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return null;
}
},
MODIFYCOLLECTION_OP(MODIFYCOLLECTION, DEFAULT_COLLECTION_OP_TIMEOUT, false) {
MODIFYCOLLECTION_OP(MODIFYCOLLECTION) {
@Override
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS.toArray(new String[0]));
Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS);
if (m.isEmpty()) throw new SolrException(ErrorCode.BAD_REQUEST,
formatString("no supported values provided rule, snitch, masShardsPerNode, replicationFactor"));
formatString("no supported values provided rule, snitch, maxShardsPerNode, replicationFactor, collection.configName"));
req.getParams().required().getAll(m, COLLECTION_PROP);
addMapObject(m, RULE);
addMapObject(m, SNITCH);
@ -1045,11 +1054,11 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
}
public static final List<String> MODIFIABLE_COLL_PROPS = ImmutableList.of(
public static final List<String> MODIFIABLE_COLL_PROPS = Arrays.asList(
RULE,
SNITCH,
REPLICATION_FACTOR,
MAX_SHARDS_PER_NODE,
AUTO_ADD_REPLICAS);
AUTO_ADD_REPLICAS,
COLL_CONF);
}

96
solr/core/src/test/org/apache/solr/cloud/OverseerModifyCollectionTest.java Normal file
View File

@ -0,0 +1,96 @@
package org.apache.solr.cloud;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import java.lang.invoke.MethodHandles;
import java.util.Map;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class OverseerModifyCollectionTest extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Test
public void testModifyColl() throws Exception {
String collName = "modifyColl";
String newConfName = "conf" + random().nextInt();
String oldConfName = "conf1";
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collName, oldConfName, 1, 2);
rsp = create.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
ConfigSetAdminRequest.Create createConfig = new ConfigSetAdminRequest.Create()
.setBaseConfigSetName(oldConfName)
.setConfigSetName(newConfName);
ConfigSetAdminResponse configRsp = createConfig.process(client);
assertEquals(0, configRsp.getStatus());
ModifiableSolrParams p = new ModifiableSolrParams();
p.add("collection", collName);
p.add("action", "MODIFYCOLLECTION");
p.add("collection.configName", newConfName);
client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
}
assertEquals(newConfName, getConfigNameFromZk(collName));
//Try an invalid config name
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
ModifiableSolrParams p = new ModifiableSolrParams();
p.add("collection", collName);
p.add("action", "MODIFYCOLLECTION");
p.add("collection.configName", "notARealConfigName");
try{
client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
fail("Exception should be thrown");
} catch(RemoteSolrException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Can not find the specified config set"));
}
}
}
private String getConfigNameFromZk(String collName) throws KeeperException, InterruptedException {
byte[] b = cloudClient.getZkStateReader().getZkClient().getData(ZkStateReader.getCollectionPathRoot(collName), null, null, false);
Map confData = (Map) Utils.fromJSON(b);
return (String) confData.get(ZkController.CONFIGNAME_PROP);
}
}

2
solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
View File

@ -107,7 +107,7 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
case "rule":
return (List) o;
default:
throw new SolrException(ErrorCode.SERVER_ERROR, "Unknown property " + propName);
return o;
}
}

6
solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
View File

@ -1075,9 +1075,13 @@ public class ZkStateReader implements Closeable {
}
}
}
public static String getCollectionPathRoot(String coll) {
return COLLECTIONS_ZKNODE+"/"+coll;
}
public static String getCollectionPath(String coll) {
return COLLECTIONS_ZKNODE+"/"+coll + "/state.json";
return getCollectionPathRoot(coll) + "/state.json";
}
/**

24
solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
View File

@ -26,6 +26,9 @@ import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@ -371,23 +374,28 @@ public abstract class SolrParams implements Serializable {
return result;
}
/**Copy all params to the given map or if the given map is null
* create a new one
*/
public Map<String, Object> getAll(Map<String, Object> sink, String... params){
if(sink == null) sink = new LinkedHashMap<>();
public Map<String, Object> getAll(Map<String, Object> sink, Collection<String> params) {
if (sink == null) sink = new LinkedHashMap<>();
for (String param : params) {
String[] v = getParams(param);
if(v != null && v.length>0 ) {
if(v.length == 1) {
if (v != null && v.length > 0) {
if (v.length == 1) {
sink.put(param, v[0]);
} else {
sink.put(param,v);
sink.put(param, v);
}
}
}
return sink;
}
/**Copy all params to the given map or if the given map is null
* create a new one
*/
public Map<String, Object> getAll(Map<String, Object> sink, String... params){
return getAll(sink, params == null ? Collections.emptyList() : Arrays.asList(params));
}
/** Returns this SolrParams as a properly URL encoded string, starting with {@code "?"}, if not empty. */
public String toQueryString() {