mirror of https://github.com/apache/lucene.git
SOLR-6512: Add a collections API call to add/delete arbitrary properties to a specific replica
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1628773 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
c48f927d10
commit
aa2f8bad25
|
@ -147,6 +147,11 @@ New Features
|
|||
|
||||
* SOLR-6476: Create a bulk mode for schema API (Noble Paul, Steve Rowe)
|
||||
|
||||
* SOLR-6512: Add a collections API call to add/delete arbitrary properties
|
||||
to a specific replica. Optionally adding sliceUnique=true will remove
|
||||
this property from all other replicas within a particular slice.
|
||||
(Erick Erickson)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
@ -36,6 +36,8 @@ import java.util.Set;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
|
@ -119,7 +121,9 @@ public class Overseer implements Closeable {
|
|||
|
||||
private static Logger log = LoggerFactory.getLogger(Overseer.class);
|
||||
|
||||
static enum LeaderStatus { DONT_KNOW, NO, YES };
|
||||
static enum LeaderStatus {DONT_KNOW, NO, YES}
|
||||
|
||||
public static final Set<String> sliceUniqueBooleanProperties = ImmutableSet.of("preferredleader");
|
||||
|
||||
private long lastUpdatedTime = 0;
|
||||
|
||||
|
@ -438,6 +442,12 @@ public class Overseer implements Closeable {
|
|||
case CLUSTERPROP:
|
||||
handleProp(message);
|
||||
break;
|
||||
case ADDREPLICAPROP:
|
||||
clusterState = addReplicaProp(clusterState, message);
|
||||
break;
|
||||
case DELETEREPLICAPROP:
|
||||
clusterState = deleteReplicaProp(clusterState, message);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("unknown operation:" + operation
|
||||
+ " contents:" + message.getProperties());
|
||||
|
@ -504,6 +514,102 @@ public class Overseer implements Closeable {
|
|||
return clusterState;
|
||||
}
|
||||
|
||||
private ClusterState addReplicaProp(ClusterState clusterState, ZkNodeProps message) {
|
||||
|
||||
if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.PROPERTY_VALUE_PROP) == false) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Overseer SETREPLICAPROPERTY requires " +
|
||||
ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " +
|
||||
ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " and " +
|
||||
ZkStateReader.PROPERTY_VALUE_PROP + " no action taken.");
|
||||
}
|
||||
|
||||
String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
|
||||
String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
|
||||
String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
|
||||
String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
|
||||
String propVal = message.getStr(ZkStateReader.PROPERTY_VALUE_PROP);
|
||||
String sliceUnique = message.getStr(OverseerCollectionProcessor.SLICE_UNIQUE);
|
||||
|
||||
boolean isUnique = false;
|
||||
|
||||
if (sliceUniqueBooleanProperties.contains(property)) {
|
||||
if (StringUtils.isNotBlank(sliceUnique) && Boolean.parseBoolean(sliceUnique) == false) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Overseer SETREPLICAPROPERTY for " +
|
||||
property + " cannot have " + OverseerCollectionProcessor.SLICE_UNIQUE + " set to anything other than" +
|
||||
"'true'. No action taken");
|
||||
}
|
||||
isUnique = true;
|
||||
} else {
|
||||
isUnique = Boolean.parseBoolean(sliceUnique);
|
||||
}
|
||||
|
||||
Replica replica = clusterState.getReplica(collectionName, replicaName);
|
||||
|
||||
if (replica == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " +
|
||||
collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
|
||||
}
|
||||
log.info("Setting property " + property + " with value: " + propVal +
|
||||
" for collection: " + collectionName + ". Full message: " + message);
|
||||
if (StringUtils.equalsIgnoreCase(replica.getStr(property), propVal)) return clusterState; // already the value we're going to set
|
||||
|
||||
// OK, there's no way we won't change the cluster state now
|
||||
Map<String,Replica> replicas = clusterState.getSlice(collectionName, sliceName).getReplicasCopy();
|
||||
if (isUnique == false) {
|
||||
replicas.get(replicaName).getProperties().put(property, propVal);
|
||||
} else { // Set prop for this replica, but remove it for all others.
|
||||
for (Replica rep : replicas.values()) {
|
||||
if (rep.getName().equalsIgnoreCase(replicaName)) {
|
||||
rep.getProperties().put(property, propVal);
|
||||
} else {
|
||||
rep.getProperties().remove(property);
|
||||
}
|
||||
}
|
||||
}
|
||||
Slice newSlice = new Slice(sliceName, replicas, clusterState.getSlice(collectionName, sliceName).shallowCopy());
|
||||
return updateSlice(clusterState, collectionName, newSlice);
|
||||
}
|
||||
|
||||
private ClusterState deleteReplicaProp(ClusterState clusterState, ZkNodeProps message) {
|
||||
|
||||
if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false ||
|
||||
checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Overseer DELETEREPLICAPROPERTY requires " +
|
||||
ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " +
|
||||
ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " no action taken.");
|
||||
}
|
||||
String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
|
||||
String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
|
||||
String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
|
||||
String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
|
||||
|
||||
Replica replica = clusterState.getReplica(collectionName, replicaName);
|
||||
|
||||
if (replica == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " +
|
||||
collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
|
||||
}
|
||||
|
||||
log.info("Deleting property " + property + " for collection: " + collectionName +
|
||||
" slice " + sliceName + " replica " + replicaName + ". Full message: " + message);
|
||||
String curProp = replica.getStr(property);
|
||||
if (curProp == null) return clusterState; // not there anyway, nothing to do.
|
||||
|
||||
Map<String, Replica> replicas = clusterState.getSlice(collectionName, sliceName).getReplicasCopy();
|
||||
replica = replicas.get(replicaName);
|
||||
replica.getProperties().remove(property);
|
||||
Slice newSlice = new Slice(sliceName, replicas, clusterState.getSlice(collectionName, sliceName).shallowCopy());
|
||||
return updateSlice(clusterState, collectionName, newSlice);
|
||||
}
|
||||
|
||||
private ClusterState setShardLeader(ClusterState clusterState, ZkNodeProps message) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String baseUrl = message.getStr(ZkStateReader.BASE_URL_PROP);
|
||||
|
@ -1055,7 +1161,6 @@ public class Overseer implements Closeable {
|
|||
newCollection = coll.copyWithSlices(slices);
|
||||
}
|
||||
|
||||
|
||||
// System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
|
||||
|
||||
return newState(state, singletonMap(collectionName, newCollection));
|
||||
|
|
|
@ -20,10 +20,12 @@ package org.apache.solr.cloud;
|
|||
import static org.apache.solr.cloud.Assign.getNodesForNewShard;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.ONLY_IF_DOWN;
|
||||
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
|
||||
|
@ -31,6 +33,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.CR
|
|||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
|
@ -51,6 +54,7 @@ import java.util.concurrent.SynchronousQueue;
|
|||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
|
@ -146,6 +150,10 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
|
||||
public static final String COLL_PROP_PREFIX = "property.";
|
||||
|
||||
public static final String ONLY_IF_DOWN = "onlyIfDown";
|
||||
|
||||
public static final String SLICE_UNIQUE = "sliceUnique";
|
||||
|
||||
public int maxParallelThreads = 10;
|
||||
|
||||
public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME);
|
||||
|
@ -619,6 +627,12 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
case CLUSTERSTATUS:
|
||||
getClusterStatus(zkStateReader.getClusterState(), message, results);
|
||||
break;
|
||||
case ADDREPLICAPROP:
|
||||
processReplicaAddPropertyCommand(message);
|
||||
break;
|
||||
case DELETEREPLICAPROP:
|
||||
processReplicaDeletePropertyCommand(message);
|
||||
break;
|
||||
default:
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
|
||||
+ operation);
|
||||
|
@ -644,6 +658,44 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
return new OverseerSolrResponse(results);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void processReplicaAddPropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException {
|
||||
if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(SHARD_ID_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(REPLICA_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(PROPERTY_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(PROPERTY_VALUE_PROP))) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST,
|
||||
String.format(Locale.ROOT, "The '%s', '%s', '%s', '%s', and '%s' parameters are required for all replica properties add/delete' operations",
|
||||
COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP, PROPERTY_VALUE_PROP));
|
||||
}
|
||||
SolrZkClient zkClient = zkStateReader.getZkClient();
|
||||
DistributedQueue inQueue = Overseer.getInQueue(zkClient);
|
||||
Map<String, Object> propMap = new HashMap<>();
|
||||
propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
|
||||
propMap.putAll(message.getProperties());
|
||||
ZkNodeProps m = new ZkNodeProps(propMap);
|
||||
inQueue.offer(ZkStateReader.toJSON(m));
|
||||
}
|
||||
|
||||
private void processReplicaDeletePropertyCommand(ZkNodeProps message) throws KeeperException, InterruptedException {
|
||||
if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(SHARD_ID_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(REPLICA_PROP)) ||
|
||||
StringUtils.isBlank(message.getStr(PROPERTY_PROP))) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST,
|
||||
String.format(Locale.ROOT, "The '%s', '%s', '%s', and '%s' parameters are required for all replica properties add/delete' operations",
|
||||
COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP));
|
||||
}
|
||||
SolrZkClient zkClient = zkStateReader.getZkClient();
|
||||
DistributedQueue inQueue = Overseer.getInQueue(zkClient);
|
||||
Map<String, Object> propMap = new HashMap<>();
|
||||
propMap.put(Overseer.QUEUE_OPERATION, DELETEREPLICAPROP.toLower());
|
||||
propMap.putAll(message.getProperties());
|
||||
ZkNodeProps m = new ZkNodeProps(propMap);
|
||||
inQueue.offer(ZkStateReader.toJSON(m));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void getOverseerStatus(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
|
||||
String leaderNode = getLeaderNode(zkStateReader.getZkClient());
|
||||
|
@ -841,6 +893,8 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
|
|||
* @param liveNodes List of currently live node names.
|
||||
* @param collectionProps Map of collection status information pulled directly from ZooKeeper.
|
||||
*/
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected void crossCheckReplicaStateWithLiveNodes(List<String> liveNodes, NamedList<Object> collectionProps) {
|
||||
Iterator<Map.Entry<String,Object>> colls = collectionProps.iterator();
|
||||
while (colls.hasNext()) {
|
||||
|
|
|
@ -21,16 +21,23 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
|
|||
import static org.apache.solr.cloud.OverseerCollectionProcessor.ASYNC;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.COLL_CONF;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATE_NODE_SET;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.SLICE_UNIQUE;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.REQUESTID;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.ONLY_IF_DOWN;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATEALIAS;
|
||||
|
@ -38,13 +45,12 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.CR
|
|||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEALIAS;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATE;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -56,6 +62,7 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
|
@ -231,6 +238,15 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
this.handleClusterStatus(req, rsp);
|
||||
break;
|
||||
}
|
||||
case ADDREPLICAPROP: {
|
||||
this.handleAddReplicaProp(req, rsp);
|
||||
break;
|
||||
}
|
||||
case DELETEREPLICAPROP: {
|
||||
this.handleDeleteReplicaProp(req, rsp);
|
||||
break;
|
||||
}
|
||||
|
||||
default: {
|
||||
throw new RuntimeException("Unknown action: " + action);
|
||||
}
|
||||
|
@ -239,6 +255,41 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
rsp.setHttpCaching(false);
|
||||
}
|
||||
|
||||
private void handleAddReplicaProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
|
||||
req.getParams().required().check(COLLECTION_PROP, PROPERTY_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_VALUE_PROP);
|
||||
|
||||
|
||||
Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
|
||||
copyIfNotNull(req.getParams(), map, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP,
|
||||
SLICE_UNIQUE, PROPERTY_VALUE_PROP);
|
||||
|
||||
String property = (String) map.get(PROPERTY_PROP);
|
||||
boolean uniquePerSlice = Boolean.parseBoolean((String) map.get(SLICE_UNIQUE));
|
||||
|
||||
// Check if we're trying to set a property with parameters that allow us to set the property on multiple replicas
|
||||
// in a slice on properties that are known to only be one-per-slice and error out if so.
|
||||
if (StringUtils.isNotBlank((String)map.get(SLICE_UNIQUE)) &&
|
||||
Overseer.sliceUniqueBooleanProperties.contains(property.toLowerCase(Locale.ROOT)) &&
|
||||
uniquePerSlice == false) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Overseer replica property command received for property " + property +
|
||||
" with the " + SLICE_UNIQUE +
|
||||
" parameter set to something other than 'true'. No action taken.");
|
||||
}
|
||||
handleResponse(ADDREPLICAPROP.toLower(), new ZkNodeProps(map), rsp);
|
||||
}
|
||||
|
||||
private void handleDeleteReplicaProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
|
||||
req.getParams().required().check(COLLECTION_PROP, PROPERTY_PROP, SHARD_ID_PROP, REPLICA_PROP);
|
||||
|
||||
Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, DELETEREPLICAPROP.toLower());
|
||||
copyIfNotNull(req.getParams(), map, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP);
|
||||
|
||||
handleResponse(DELETEREPLICAPROP.toLower(), new ZkNodeProps(map), rsp);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void handleOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
|
||||
Map<String, Object> props = ZkNodeProps.makeMap(
|
||||
Overseer.QUEUE_OPERATION, OVERSEERSTATUS.toLower());
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.solr.cloud;
|
|||
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
|
||||
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
|
||||
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
|
||||
|
||||
|
@ -153,7 +154,7 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
|
|||
"action", DELETEREPLICA.toLower(),
|
||||
"shard", shard,
|
||||
"replica", replica.getName(),
|
||||
ZkStateReader.ONLY_IF_DOWN, "true");
|
||||
ONLY_IF_DOWN, "true");
|
||||
SolrParams params = new MapSolrParams(m);
|
||||
SolrRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
|
|
|
@ -19,20 +19,29 @@ package org.apache.solr.cloud;
|
|||
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.CloudSolrServer;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.params.CollectionParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.junit.Before;
|
||||
|
||||
import static org.apache.solr.cloud.OverseerCollectionProcessor.SLICE_UNIQUE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
|
||||
|
@ -57,7 +66,7 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
|
|||
public void doTest() throws Exception {
|
||||
CloudSolrServer client = createCloudClient(null);
|
||||
try {
|
||||
createCollection(null, COLLECTION_NAME, 2, 1, 1, client, null, "conf1");
|
||||
createCollection(null, COLLECTION_NAME, 2, 2, 2, client, null, "conf1");
|
||||
createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
|
||||
} finally {
|
||||
//remove collections
|
||||
|
@ -76,6 +85,7 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
|
|||
clusterStatusWithRouteKey();
|
||||
clusterStatusAliasTest();
|
||||
clusterStatusRolesTest();
|
||||
replicaPropTest();
|
||||
}
|
||||
|
||||
private void clusterStatusWithCollectionAndShard() throws IOException, SolrServerException {
|
||||
|
@ -281,4 +291,299 @@ public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
|
|||
client.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private void replicaPropTest() throws Exception {
|
||||
CloudSolrServer client = createCloudClient(null);
|
||||
try {
|
||||
client.connect();
|
||||
Map<String, Slice> slices = client.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlicesMap();
|
||||
List<String> sliceList = new ArrayList<>(slices.keySet());
|
||||
String c1_s1 = sliceList.get(0);
|
||||
List<String> replicasList = new ArrayList<>(slices.get(c1_s1).getReplicasMap().keySet());
|
||||
String c1_s1_r1 = replicasList.get(0);
|
||||
String c1_s1_r2 = replicasList.get(1);
|
||||
|
||||
String c1_s2 = sliceList.get(1);
|
||||
replicasList = new ArrayList<>(slices.get(c1_s2).getReplicasMap().keySet());
|
||||
String c1_s2_r1 = replicasList.get(0);
|
||||
String c1_s2_r2 = replicasList.get(1);
|
||||
|
||||
|
||||
slices = client.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME1).getSlicesMap();
|
||||
sliceList = new ArrayList<>(slices.keySet());
|
||||
String c2_s1 = sliceList.get(0);
|
||||
replicasList = new ArrayList<>(slices.get(c2_s1).getReplicasMap().keySet());
|
||||
String c2_s1_r1 = replicasList.get(0);
|
||||
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString());
|
||||
|
||||
// Insure we get error returns when omitting required parameters
|
||||
|
||||
missingParamsError(client, params);
|
||||
params.set("collection", COLLECTION_NAME);
|
||||
missingParamsError(client, params);
|
||||
params.set("shard", c1_s1);
|
||||
missingParamsError(client, params);
|
||||
params.set("replica", c1_s1_r1);
|
||||
missingParamsError(client, params);
|
||||
params.set("property", "preferredLeader");
|
||||
missingParamsError(client, params);
|
||||
params.set("property.value", "true");
|
||||
|
||||
SolrRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
client.request(request);
|
||||
|
||||
// The above should have set exactly one preferredleader...
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r2, "preferredLeader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r1, "preferredLeader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r2,
|
||||
"property", "preferredLeader",
|
||||
"property.value", "true");
|
||||
// The preferred leader property for shard1 should have switched to the other replica.
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredLeader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r1, "preferredLeader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s2,
|
||||
"replica", c1_s2_r1,
|
||||
"property", "preferredLeader",
|
||||
"property.value", "true");
|
||||
|
||||
// Now we should have a preferred leader in both shards...
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredLeader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME1,
|
||||
"shard", c2_s1,
|
||||
"replica", c2_s1_r1,
|
||||
"property", "preferredLeader",
|
||||
"property.value", "true");
|
||||
|
||||
// Now we should have three preferred leaders.
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader", "true");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.DELETEREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME1,
|
||||
"shard", c2_s1,
|
||||
"replica", c2_s1_r1,
|
||||
"property", "preferredLeader");
|
||||
|
||||
// Now we should have two preferred leaders.
|
||||
// But first we have to wait for the overseer to finish the action
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
// Try adding an arbitrary property to one that has the leader property
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r1,
|
||||
"property", "testprop",
|
||||
"property.value", "true");
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r2,
|
||||
"property", "prop",
|
||||
"property.value", "silly");
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toLower(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r1,
|
||||
"property", "testprop",
|
||||
"property.value", "nonsense",
|
||||
SLICE_UNIQUE, "true");
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "nonsense");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toLower(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r1,
|
||||
"property", "testprop",
|
||||
"property.value", "true",
|
||||
SLICE_UNIQUE, "false");
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r1, "testprop", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.DELETEREPLICAPROP.toLower(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r1,
|
||||
"property", "testprop");
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "testprop");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
try {
|
||||
doPropertyAction(client,
|
||||
"action", CollectionParams.CollectionAction.ADDREPLICAPROP.toString(),
|
||||
"collection", COLLECTION_NAME,
|
||||
"shard", c1_s1,
|
||||
"replica", c1_s1_r1,
|
||||
"property", "preferredLeader",
|
||||
"property.value", "true",
|
||||
SLICE_UNIQUE, "false");
|
||||
fail("Should have thrown an exception, setting sliceUnique=false is not allowed for 'preferredLeader'.");
|
||||
} catch (SolrException se) {
|
||||
assertTrue("Should have received a specific error message",
|
||||
se.getMessage().contains("with the sliceUnique parameter set to something other than 'true'"));
|
||||
}
|
||||
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "preferredleader", "true");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s2_r1, "preferredleader", "true");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "testprop");
|
||||
verifyPropertyVal(client, COLLECTION_NAME, c1_s1_r2, "prop", "silly");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME, c1_s2_r2, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
verifyPropertyNotPresent(client, COLLECTION_NAME1, c2_s1_r1, "preferredleader");
|
||||
|
||||
} finally {
|
||||
client.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
private void doPropertyAction(CloudSolrServer client, String... paramsIn) throws IOException, SolrServerException {
|
||||
assertTrue("paramsIn must be an even multiple of 2, it is: " + paramsIn.length, (paramsIn.length % 2) == 0);
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
for (int idx = 0; idx < paramsIn.length; idx += 2) {
|
||||
params.set(paramsIn[idx], paramsIn[idx + 1]);
|
||||
}
|
||||
QueryRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
client.request(request);
|
||||
|
||||
}
|
||||
|
||||
private void verifyPropertyNotPresent(CloudSolrServer client, String collectionName, String replicaName,
|
||||
String property)
|
||||
throws KeeperException, InterruptedException {
|
||||
ClusterState clusterState = null;
|
||||
Replica replica = null;
|
||||
for (int idx = 0; idx < 300; ++idx) {
|
||||
client.getZkStateReader().updateClusterState(true);
|
||||
clusterState = client.getZkStateReader().getClusterState();
|
||||
replica = clusterState.getReplica(collectionName, replicaName);
|
||||
if (replica == null) {
|
||||
fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
|
||||
}
|
||||
if (StringUtils.isBlank(replica.getStr(property))) return;
|
||||
Thread.sleep(100);
|
||||
}
|
||||
fail("Property " + property + " not set correctly for collection/replica pair: " +
|
||||
collectionName + "/" + replicaName + ". Replica props: " + replica.getProperties().toString() +
|
||||
". Cluster state is " + clusterState.toString());
|
||||
|
||||
}
|
||||
|
||||
// The params are triplets,
|
||||
// collection
|
||||
// shard
|
||||
// replica
|
||||
private void verifyPropertyVal(CloudSolrServer client, String collectionName,
|
||||
String replicaName, String property, String val)
|
||||
throws InterruptedException, KeeperException {
|
||||
Replica replica = null;
|
||||
ClusterState clusterState = null;
|
||||
|
||||
for (int idx = 0; idx < 300; ++idx) { // Keep trying while Overseer writes the ZK state for up to 30 seconds.
|
||||
client.getZkStateReader().updateClusterState(true);
|
||||
clusterState = client.getZkStateReader().getClusterState();
|
||||
replica = clusterState.getReplica(collectionName, replicaName);
|
||||
if (replica == null) {
|
||||
fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
|
||||
}
|
||||
if (StringUtils.equals(val, replica.getStr(property))) return;
|
||||
Thread.sleep(100);
|
||||
}
|
||||
|
||||
fail("Property '" + property + "' with value " + replica.getStr(property) +
|
||||
" not set correctly for collection/replica pair: " + collectionName + "/" + replicaName + " property map is " +
|
||||
replica.getProperties().toString() + ".");
|
||||
|
||||
}
|
||||
|
||||
private void missingParamsError(CloudSolrServer client, ModifiableSolrParams origParams)
|
||||
throws IOException, SolrServerException {
|
||||
|
||||
SolrRequest request;
|
||||
try {
|
||||
request = new QueryRequest(origParams);
|
||||
request.setPath("/admin/collections");
|
||||
client.request(request);
|
||||
fail("Should have thrown a SolrException due to lack of a required parameter.");
|
||||
} catch (SolrException se) {
|
||||
assertTrue("Should have gotten a specific message back mentioning 'missing required parameter'. Got: " + se.getMessage(),
|
||||
se.getMessage().toLowerCase(Locale.ROOT).contains("missing required parameter:"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -69,6 +69,8 @@ public class ZkStateReader implements Closeable {
|
|||
public static final String SHARD_PARENT_PROP = "shard_parent";
|
||||
public static final String NUM_SHARDS_PROP = "numShards";
|
||||
public static final String LEADER_PROP = "leader";
|
||||
public static final String PROPERTY_PROP = "property";
|
||||
public static final String PROPERTY_VALUE_PROP = "property.value";
|
||||
|
||||
public static final String COLLECTIONS_ZKNODE = "/collections";
|
||||
public static final String LIVE_NODES_ZKNODE = "/live_nodes";
|
||||
|
@ -102,8 +104,7 @@ public class ZkStateReader implements Closeable {
|
|||
public static final String LEADER_ELECT_ZKNODE = "/leader_elect";
|
||||
|
||||
public static final String SHARD_LEADERS_ZKNODE = "leaders";
|
||||
public static final String ONLY_IF_DOWN = "onlyIfDown";
|
||||
|
||||
|
||||
private final Set<String> watchedCollections = new HashSet<String>();
|
||||
|
||||
/**These are collections which are actively watched by this instance .
|
||||
|
|
|
@ -46,7 +46,9 @@ public interface CollectionParams
|
|||
ADDREPLICA,
|
||||
OVERSEERSTATUS,
|
||||
LIST,
|
||||
CLUSTERSTATUS;
|
||||
CLUSTERSTATUS,
|
||||
ADDREPLICAPROP,
|
||||
DELETEREPLICAPROP;
|
||||
|
||||
public static CollectionAction get( String p )
|
||||
{
|
||||
|
|
Loading…
Reference in New Issue