SOLR-6512: Add a collections API call to add/delete arbitrary properties to a specific replica

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1628773 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erick Erickson 2014-10-01 16:57:49 +00:00
parent c48f927d10
commit aa2f8bad25
8 changed files with 535 additions and 11 deletions

View File

@ -147,6 +147,11 @@ New Features
* SOLR-6476: Create a bulk mode for schema API (Noble Paul, Steve Rowe)
* SOLR-6512: Add a collections API call to add/delete arbitrary properties
to a specific replica. Optionally adding sliceUnique=true will remove
this property from all other replicas within a particular slice.
(Erick Erickson)
Bug Fixes
----------------------

View File

@ -36,6 +36,8 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
@ -119,7 +121,9 @@ public class Overseer implements Closeable {
private static Logger log = LoggerFactory.getLogger(Overseer.class);
static enum LeaderStatus { DONT_KNOW, NO, YES };
static enum LeaderStatus {DONT_KNOW, NO, YES}
public static final Set<String> sliceUniqueBooleanProperties = ImmutableSet.of("preferredleader");
private long lastUpdatedTime = 0;
@ -438,6 +442,12 @@ public class Overseer implements Closeable {
case CLUSTERPROP:
handleProp(message);
break;
case ADDREPLICAPROP:
clusterState = addReplicaProp(clusterState, message);
break;
case DELETEREPLICAPROP:
clusterState = deleteReplicaProp(clusterState, message);
break;
default:
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
@ -504,6 +514,102 @@ public class Overseer implements Closeable {
return clusterState;
}
private ClusterState addReplicaProp(ClusterState clusterState, ZkNodeProps message) {
if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false ||
checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false ||
checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false ||
checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false ||
checkKeyExistence(message, ZkStateReader.PROPERTY_VALUE_PROP) == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Overseer SETREPLICAPROPERTY requires " +
ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " +
ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " and " +
ZkStateReader.PROPERTY_VALUE_PROP + " no action taken.");
}
String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
String propVal = message.getStr(ZkStateReader.PROPERTY_VALUE_PROP);
String sliceUnique = message.getStr(OverseerCollectionProcessor.SLICE_UNIQUE);
boolean isUnique = false;
if (sliceUniqueBooleanProperties.contains(property)) {
if (StringUtils.isNotBlank(sliceUnique) && Boolean.parseBoolean(sliceUnique) == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Overseer SETREPLICAPROPERTY for " +
property + " cannot have " + OverseerCollectionProcessor.SLICE_UNIQUE + " set to anything other than" +
"'true'. No action taken");
}
isUnique = true;
} else {
isUnique = Boolean.parseBoolean(sliceUnique);
}
Replica replica = clusterState.getReplica(collectionName, replicaName);
if (replica == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " +
collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
}
log.info("Setting property " + property + " with value: " + propVal +
" for collection: " + collectionName + ". Full message: " + message);
if (StringUtils.equalsIgnoreCase(replica.getStr(property), propVal)) return clusterState; // already the value we're going to set
// OK, there's no way we won't change the cluster state now
Map<String,Replica> replicas = clusterState.getSlice(collectionName, sliceName).getReplicasCopy();
if (isUnique == false) {
replicas.get(replicaName).getProperties().put(property, propVal);
} else { // Set prop for this replica, but remove it for all others.
for (Replica rep : replicas.values()) {
if (rep.getName().equalsIgnoreCase(replicaName)) {
rep.getProperties().put(property, propVal);
} else {
rep.getProperties().remove(property);
}
}
}
Slice newSlice = new Slice(sliceName, replicas, clusterState.getSlice(collectionName, sliceName).shallowCopy());
return updateSlice(clusterState, collectionName, newSlice);
}
private ClusterState deleteReplicaProp(ClusterState clusterState, ZkNodeProps message) {
if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false ||
checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false ||
checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false ||
checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Overseer DELETEREPLICAPROPERTY requires " +
ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " +
ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " no action taken.");
}
String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
Replica replica = clusterState.getReplica(collectionName, replicaName);
if (replica == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " +
collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
}
log.info("Deleting property " + property + " for collection: " + collectionName +
" slice " + sliceName + " replica " + replicaName + ". Full message: " + message);
String curProp = replica.getStr(property);
if (curProp == null) return clusterState; // not there anyway, nothing to do.
Map<String, Replica> replicas = clusterState.getSlice(collectionName, sliceName).getReplicasCopy();
replica = replicas.get(replicaName);
replica.getProperties().remove(property);
Slice newSlice = new Slice(sliceName, replicas, clusterState.getSlice(collectionName, sliceName).shallowCopy());
return updateSlice(clusterState, collectionName, newSlice);
}
private ClusterState setShardLeader(ClusterState clusterState, ZkNodeProps message) {
StringBuilder sb = new StringBuilder();
String baseUrl = message.getStr(ZkStateReader.BASE_URL_PROP);
@ -1055,7 +1161,6 @@ public class Overseer implements Closeable {
newCollection = coll.copyWithSlices(slices);
}
// System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
return newState(state, singletonMap(collectionName, newCollection));

View File

@ -20,10 +20,12 @@ package org.apache.solr.cloud;
import static org.apache.solr.cloud.Assign.getNodesForNewShard;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.ONLY_IF_DOWN;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
@ -31,6 +33,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.CR
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
import java.io.Closeable;
import java.io.IOException;
@ -51,6 +54,7 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@ -146,6 +150,10 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
public static final String COLL_PROP_PREFIX = "property.";
public static final String ONLY_IF_DOWN = "onlyIfDown";
public static final String SLICE_UNIQUE = "sliceUnique";
public int maxParallelThreads = 10;
public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME);
@ -619,6 +627,12 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
case CLUSTERSTATUS:
getClusterStatus(zkStateReader.getClusterState(), message, results);
break;
case ADDREPLICAPROP:
processReplicaAddPropertyCommand(message);
break;
case DELETEREPLICAPROP:
processReplicaDeletePropertyCommand(message);
break;
default:
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
@ -644,6 +658,44 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
return new OverseerSolrResponse(results);
}
@SuppressWarnings("unchecked")
private void processReplicaAddPropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException {
if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) ||
StringUtils.isBlank(message.getStr(SHARD_ID_PROP)) ||
StringUtils.isBlank(message.getStr(REPLICA_PROP)) ||
StringUtils.isBlank(message.getStr(PROPERTY_PROP)) ||
StringUtils.isBlank(message.getStr(PROPERTY_VALUE_PROP))) {
throw new SolrException(ErrorCode.BAD_REQUEST,
String.format(Locale.ROOT, "The '%s', '%s', '%s', '%s', and '%s' parameters are required for all replica properties add/delete' operations",
COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP, PROPERTY_VALUE_PROP));
}
SolrZkClient zkClient = zkStateReader.getZkClient();
DistributedQueue inQueue = Overseer.getInQueue(zkClient);
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
propMap.putAll(message.getProperties());
ZkNodeProps m = new ZkNodeProps(propMap);
inQueue.offer(ZkStateReader.toJSON(m));
}
private void processReplicaDeletePropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException {
if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) ||
StringUtils.isBlank(message.getStr(SHARD_ID_PROP)) ||
StringUtils.isBlank(message.getStr(REPLICA_PROP)) ||
StringUtils.isBlank(message.getStr(PROPERTY_PROP))) {
throw new SolrException(ErrorCode.BAD_REQUEST,
String.format(Locale.ROOT, "The '%s', '%s', '%s', and '%s' parameters are required for all replica properties add/delete' operations",
COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP));
}
SolrZkClient zkClient = zkStateReader.getZkClient();
DistributedQueue inQueue = Overseer.getInQueue(zkClient);
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, DELETEREPLICAPROP.toLower());
propMap.putAll(message.getProperties());
ZkNodeProps m = new ZkNodeProps(propMap);
inQueue.offer(ZkStateReader.toJSON(m));
}
@SuppressWarnings("unchecked")
private void getOverseerStatus(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
String leaderNode = getLeaderNode(zkStateReader.getZkClient());
@ -841,6 +893,8 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
* @param liveNodes List of currently live node names.
* @param collectionProps Map of collection status information pulled directly from ZooKeeper.
*/
@SuppressWarnings("unchecked")
protected void crossCheckReplicaStateWithLiveNodes(List<String> liveNodes, NamedList<Object> collectionProps) {
Iterator<Map.Entry<String,Object>> colls = collectionProps.iterator();
while (colls.hasNext()) {

View File

@ -21,16 +21,23 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ASYNC;
import static org.apache.solr.cloud.OverseerCollectionProcessor.COLL_CONF;
import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATE_NODE_SET;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SLICE_UNIQUE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REQUESTID;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.ONLY_IF_DOWN;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATEALIAS;
@ -38,13 +45,12 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.CR
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEALIAS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
import java.io.IOException;
@ -56,6 +62,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@ -231,6 +238,15 @@ public class CollectionsHandler extends RequestHandlerBase {
this.handleClusterStatus(req, rsp);
break;
}
case ADDREPLICAPROP: {
this.handleAddReplicaProp(req, rsp);
break;
}
case DELETEREPLICAPROP: {
this.handleDeleteReplicaProp(req, rsp);
break;
}
default: {
throw new RuntimeException("Unknown action: " + action);
}
@ -239,6 +255,41 @@ public class CollectionsHandler extends RequestHandlerBase {
rsp.setHttpCaching(false);
}
private void handleAddReplicaProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
req.getParams().required().check(COLLECTION_PROP, PROPERTY_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_VALUE_PROP);
Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
copyIfNotNull(req.getParams(), map, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP,
SLICE_UNIQUE, PROPERTY_VALUE_PROP);
String property = (String) map.get(PROPERTY_PROP);
boolean uniquePerSlice = Boolean.parseBoolean((String) map.get(SLICE_UNIQUE));
// Check if we're trying to set a property with parameters that allow us to set the property on multiple replicas
// in a slice on properties that are known to only be one-per-slice and error out if so.
if (StringUtils.isNotBlank((String)map.get(SLICE_UNIQUE)) &&
Overseer.sliceUniqueBooleanProperties.contains(property.toLowerCase(Locale.ROOT)) &&
uniquePerSlice == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Overseer replica property command received for property " + property +
" with the " + SLICE_UNIQUE +
" parameter set to something other than 'true'. No action taken.");
}
handleResponse(ADDREPLICAPROP.toLower(), new ZkNodeProps(map), rsp);
}
private void handleDeleteReplicaProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
req.getParams().required().check(COLLECTION_PROP, PROPERTY_PROP, SHARD_ID_PROP, REPLICA_PROP);
Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, DELETEREPLICAPROP.toLower());
copyIfNotNull(req.getParams(), map, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP);
handleResponse(DELETEREPLICAPROP.toLower(), new ZkNodeProps(map), rsp);
}
private void handleOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
Map<String, Object> props = ZkNodeProps.makeMap(
Overseer.QUEUE_OPERATION, OVERSEERSTATUS.toLower());

View File

@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
@ -153,7 +154,7 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
"action", DELETEREPLICA.toLower(),
"shard", shard,
"replica", replica.getName(),
ZkStateReader.ONLY_IF_DOWN, "true");
ONLY_IF_DOWN, "true");
SolrParams params = new MapSolrParams(m);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");

View File

@ -19,20 +19,29 @@ package org.apache.solr.cloud;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
import org.apache.zookeeper.KeeperException;
import org.junit.Before;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SLICE_UNIQUE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
@ -57,7 +66,7 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
public void doTest() throws Exception {
CloudSolrServer client = createCloudClient(null);
try {
createCollection(null, COLLECTION_NAME, 2, 1, 1, client, null, "conf1");
createCollection(null, COLLECTION_NAME, 2, 2, 2, client, null, "conf1");
createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
} finally {
//remove collections
@ -76,6 +85,7 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
clusterStatusWithRouteKey();
clusterStatusAliasTest();
clusterStatusRolesTest();
replicaPropTest();
}
private void clusterStatusWithCollectionAndShard() throws IOException, SolrServerException {
@ -281,4 +291,299 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
client.shutdown();
}
}
private void replicaPropTest() throws Exception {
CloudSolrServer client = createCloudClient(null);
try {
client.connect();
Map<String, Slice> slices = client.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlicesMap();
List<String> sliceList = new ArrayList<>(slices.keySet());
String c1_s1 = sliceList.get(0);
List<String> replicasList = new ArrayList<>(slices.get(c1_s1).getReplicasMap().keySet());
String c1_s1_r1 = replicasList.get(0);
String c1_s1_r2 = replicasList.get(1);
String c1_s2 = sliceList.get(1);
replicasList = new ArrayList<>(slices.get(c1_s2).getReplicasMap().keySet());
String c1_s2_r1 = replicasList.get(0);
String c1_s2_r2 = replicasList.get(1);
slices = client.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME1).getSlicesMap();
sliceList = new ArrayList<>(slices.keySet());
String c2_s1 = sliceList.get(0);
replicasList = new ArrayList<>(slices.get(c2_s1).getReplicasMap().keySet());
String c2_s1_r1 = replicasList.get(0);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString());
// Insure we get error returns when omitting required parameters
missingParamsError(client, params);
params.set("collection", COLLECTION_NAME);
missingParamsError(client, params);
params.set("shard", c1_s1);
missingParamsError(client, params);
params.set("replica", c1_s1_r1);
missingParamsError(client, params);
params.set("property", "preferredLeader");
missingParamsError(client, params);
params.set("property.value", "true");
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
// The above should have set exactly one preferredleader...
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r2, "preferredLeader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r1, "preferredLeader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r2,
"property", "preferredLeader",
"property.value", "true");
// The preferred leader property for shard1 should have switched to the other replica.
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredLeader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r1, "preferredLeader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME,
"shard", c1_s2,
"replica", c1_s2_r1,
"property", "preferredLeader",
"property.value", "true");
// Now we should have a preferred leader in both shards...
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME1,
"shard", c2_s1,
"replica", c2_s1_r1,
"property", "preferredLeader",
"property.value", "true");
// Now we should have three preferred leaders.
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader", "true");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.DELETEREPLICAPROP.toString(),
"collection", COLLECTION_NAME1,
"shard", c2_s1,
"replica", c2_s1_r1,
"property", "preferredLeader");
// Now we should have two preferred leaders.
// But first we have to wait for the overseer to finish the action
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
// Try adding an arbitrary property to one that has the leader property
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r1,
"property", "testprop",
"property.value", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r2,
"property", "prop",
"property.value", "silly");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toLower(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r1,
"property", "testprop",
"property.value", "nonsense",
SLICE_UNIQUE, "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "nonsense");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toLower(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r1,
"property", "testprop",
"property.value", "true",
SLICE_UNIQUE, "false");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
doPropertyAction(client,
"action", CollectionParams.CollectionAction.DELETEREPLICAPROP.toLower(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r1,
"property", "testprop");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "testprop");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
try {
doPropertyAction(client,
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
"collection", COLLECTION_NAME,
"shard", c1_s1,
"replica", c1_s1_r1,
"property", "preferredLeader",
"property.value", "true",
SLICE_UNIQUE, "false");
fail("Should have thrown an exception, setting sliceUnique=false is not allowed for 'preferredLeader'.");
} catch (SolrException se) {
assertTrue("Should have received a specific error message",
se.getMessage().contains("with the sliceUnique parameter set to something other than 'true'"));
}
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "testprop");
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
} finally {
client.shutdown();
}
}
private void doPropertyAction(CloudSolrServer client, String... paramsIn) throws IOException, SolrServerException {
assertTrue("paramsIn must be an even multiple of 2, it is: " + paramsIn.length, (paramsIn.length % 2) == 0);
ModifiableSolrParams params = new ModifiableSolrParams();
for (int idx = 0; idx < paramsIn.length; idx += 2) {
params.set(paramsIn[idx], paramsIn[idx + 1]);
}
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
}
private void verifyPropertyNotPresent(CloudSolrServer client, String collectionName, String replicaName,
String property)
throws KeeperException, InterruptedException {
ClusterState clusterState = null;
Replica replica = null;
for (int idx = 0; idx < 300; ++idx) {
client.getZkStateReader().updateClusterState(true);
clusterState = client.getZkStateReader().getClusterState();
replica = clusterState.getReplica(collectionName, replicaName);
if (replica == null) {
fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
}
if (StringUtils.isBlank(replica.getStr(property))) return;
Thread.sleep(100);
}
fail("Property " + property + " not set correctly for collection/replica pair: " +
collectionName + "/" + replicaName + ". Replica props: " + replica.getProperties().toString() +
". Cluster state is " + clusterState.toString());
}
// The params are triplets,
// collection
// shard
// replica
private void verifyPropertyVal(CloudSolrServer client, String collectionName,
String replicaName, String property, String val)
throws InterruptedException, KeeperException {
Replica replica = null;
ClusterState clusterState = null;
for (int idx = 0; idx < 300; ++idx) { // Keep trying while Overseer writes the ZK state for up to 30 seconds.
client.getZkStateReader().updateClusterState(true);
clusterState = client.getZkStateReader().getClusterState();
replica = clusterState.getReplica(collectionName, replicaName);
if (replica == null) {
fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
}
if (StringUtils.equals(val, replica.getStr(property))) return;
Thread.sleep(100);
}
fail("Property '" + property + "' with value " + replica.getStr(property) +
" not set correctly for collection/replica pair: " + collectionName + "/" + replicaName + " property map is " +
replica.getProperties().toString() + ".");
}
private void missingParamsError(CloudSolrServer client, ModifiableSolrParams origParams)
throws IOException, SolrServerException {
SolrRequest request;
try {
request = new QueryRequest(origParams);
request.setPath("/admin/collections");
client.request(request);
fail("Should have thrown a SolrException due to lack of a required parameter.");
} catch (SolrException se) {
assertTrue("Should have gotten a specific message back mentioning 'missing required parameter'. Got: " + se.getMessage(),
se.getMessage().toLowerCase(Locale.ROOT).contains("missing required parameter:"));
}
}
}

View File

@ -69,6 +69,8 @@ public class ZkStateReader implements Closeable {
public static final String SHARD_PARENT_PROP = "shard_parent";
public static final String NUM_SHARDS_PROP = "numShards";
public static final String LEADER_PROP = "leader";
public static final String PROPERTY_PROP = "property";
public static final String PROPERTY_VALUE_PROP = "property.value";
public static final String COLLECTIONS_ZKNODE = "/collections";
public static final String LIVE_NODES_ZKNODE = "/live_nodes";
@ -102,8 +104,7 @@ public class ZkStateReader implements Closeable {
public static final String LEADER_ELECT_ZKNODE = "/leader_elect";
public static final String SHARD_LEADERS_ZKNODE = "leaders";
public static final String ONLY_IF_DOWN = "onlyIfDown";
private final Set<String> watchedCollections = new HashSet<String>();
/**These are collections which are actively watched by this instance .

View File

@ -46,7 +46,9 @@ public interface CollectionParams
ADDREPLICA,
OVERSEERSTATUS,
LIST,
CLUSTERSTATUS;
CLUSTERSTATUS,
ADDREPLICAPROP,
DELETEREPLICAPROP;
public static CollectionAction get( String p )
{