Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14964: remove Autoscaling related withCollection and COLOCATED_WITH (#2034) 

Part of SOLR-14656 (remove Autoscalin)
This commit is contained in:
Ilan Ginzburg 2020-10-26 19:10:49 +01:00 committed by GitHub
parent c29b0083d7
commit b649f3f40d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 3 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
View File

@ -28,7 +28,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT;
@ -61,7 +60,6 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
@ -202,29 +200,6 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
}
private ModifiableSolrParams getReplicaParams(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results, String collectionName, DocCollection coll, boolean skipCreateReplicaInClusterState, String asyncId, ShardHandler shardHandler, CreateReplica createReplica) throws IOException, InterruptedException, KeeperException {
if (coll.getStr(WITH_COLLECTION) != null) {
String withCollectionName = coll.getStr(WITH_COLLECTION);
DocCollection withCollection = clusterState.getCollection(withCollectionName);
if (withCollection.getActiveSlices().size() > 1) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + withCollection.getActiveSlices().size());
}
String withCollectionShard = withCollection.getActiveSlices().iterator().next().getName();
List<Replica> replicas = withCollection.getReplicas(createReplica.node);
if (replicas == null || replicas.isEmpty()) {
// create a replica of withCollection on the identified node before proceeding further
ZkNodeProps props = new ZkNodeProps(
Overseer.QUEUE_OPERATION, ADDREPLICA.toString(),
ZkStateReader.COLLECTION_PROP, withCollectionName,
ZkStateReader.SHARD_ID_PROP, withCollectionShard,
"node", createReplica.node,
// since we already computed node assignments (which include assigning a node for this withCollection replica) we want to skip the assignment step
CollectionAdminParams.SKIP_NODE_ASSIGNMENT, "true",
CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created
addReplica(clusterState, props, results, null);
}
}
ModifiableSolrParams params = new ModifiableSolrParams();
ZkStateReader zkStateReader = ocmh.zkStateReader;

50
solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
View File

@ -30,7 +30,6 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.cloud.AlreadyExistsException;
import org.apache.solr.client.solrj.cloud.BadVersionException;
@ -78,9 +77,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
import static org.apache.solr.common.params.CommonParams.NAME;
@ -118,21 +115,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection alias already exists: " + collectionName);
}
String withCollection = message.getStr(CollectionAdminParams.WITH_COLLECTION);
String withCollectionShard = null;
if (withCollection != null) {
String realWithCollection = aliases.resolveSimpleAlias(withCollection);
if (!clusterState.hasCollection(realWithCollection)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "The 'withCollection' does not exist: " + realWithCollection);
} else {
DocCollection collection = clusterState.getCollection(realWithCollection);
if (collection.getActiveSlices().size() > 1) {
throw new SolrException(ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + collection.getActiveSlices().size());
}
withCollectionShard = collection.getActiveSlices().iterator().next().getName();
}
}
String configName = getConfigName(collectionName, message);
if (configName == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");
@ -209,22 +191,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
for (ReplicaPosition replicaPosition : replicaPositions) {
String nodeName = replicaPosition.node;
if (withCollection != null) {
// check that we have a replica of `withCollection` on this node and if not, create one
DocCollection collection = clusterState.getCollection(withCollection);
List<Replica> replicas = collection.getReplicas(nodeName);
if (replicas == null || replicas.isEmpty()) {
ZkNodeProps props = new ZkNodeProps(
Overseer.QUEUE_OPERATION, ADDREPLICA.toString(),
ZkStateReader.COLLECTION_PROP, withCollection,
ZkStateReader.SHARD_ID_PROP, withCollectionShard,
"node", nodeName,
CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created
new AddReplicaCmd(ocmh).call(clusterState, props, results);
clusterState = zkStateReader.getClusterState(); // refresh
}
}
String coreName = Assign.buildSolrCoreName(ocmh.cloudManager.getDistribStateManager(),
ocmh.cloudManager.getClusterStateProvider().getClusterState().getCollection(collectionName),
replicaPosition.shard, replicaPosition.type, true);
@ -309,22 +275,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
}
// modify the `withCollection` and store this new collection's name with it
if (withCollection != null) {
ZkNodeProps props = new ZkNodeProps(
Overseer.QUEUE_OPERATION, MODIFYCOLLECTION.toString(),
ZkStateReader.COLLECTION_PROP, withCollection,
CollectionAdminParams.COLOCATED_WITH, collectionName);
ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
try {
zkStateReader.waitForState(withCollection, 5, TimeUnit.SECONDS, (collectionState) -> collectionName.equals(collectionState.getStr(COLOCATED_WITH)));
} catch (TimeoutException e) {
log.warn("Timed out waiting to see the {} property set on collection: {}", COLOCATED_WITH, withCollection);
// maybe the overseer queue is backed up, we don't want to fail the create request
// because of this time out, continue
}
}
// create an alias pointing to the new collection, if different from the collectionName
if (!alias.equals(collectionName)) {
ocmh.zkStateReader.aliasesManager.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(alias, collectionName));

21
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
View File

@ -52,9 +52,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
@ -94,8 +92,6 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
collection = extCollection;
}
checkNotColocatedWith(zkStateReader, collection);
final boolean deleteHistory = message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true);
boolean removeCounterNode = true;
@ -262,21 +258,4 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
.map(Map.Entry::getKey) // alias name
.collect(Collectors.toList());
}
private void checkNotColocatedWith(ZkStateReader zkStateReader, String collection) throws Exception {
DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collection);
if (docCollection != null) {
String colocatedWith = docCollection.getStr(COLOCATED_WITH);
if (colocatedWith != null) {
DocCollection colocatedCollection = zkStateReader.getClusterState().getCollectionOrNull(colocatedWith);
if (colocatedCollection != null && collection.equals(colocatedCollection.getStr(WITH_COLLECTION))) {
// todo how do we clean up if reverse-link is not present?
// can't delete this collection because it is still co-located with another collection
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collection: " + collection + " is co-located with collection: " + colocatedWith
+ " remove the link using modify collection API or delete the co-located collection: " + colocatedWith);
}
}
}
}
}

21
solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
View File

@ -33,7 +33,6 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
@ -126,26 +125,6 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
replica = sliceReplicas.iterator().next();
}
if (coll.getStr(CollectionAdminParams.COLOCATED_WITH) != null) {
// we must ensure that moving this replica does not cause the co-location to break
String sourceNode = replica.getNodeName();
String colocatedCollectionName = coll.getStr(CollectionAdminParams.COLOCATED_WITH);
DocCollection colocatedCollection = clusterState.getCollectionOrNull(colocatedCollectionName);
if (colocatedCollection != null) {
if (colocatedCollection.getReplica((s, r) -> sourceNode.equals(r.getNodeName())) != null) {
// check if we have at least two replicas of the collection on the source node
// only then it is okay to move one out to another node
List<Replica> replicasOnSourceNode = coll.getReplicas(replica.getNodeName());
if (replicasOnSourceNode == null || replicasOnSourceNode.size() < 2) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collection: " + collection + " is co-located with collection: " + colocatedCollectionName
+ " and has a single replica: " + replica.getName() + " on node: " + replica.getNodeName()
+ " so it is not possible to move it to another node");
}
}
}
}
log.info("Replica will be moved to node {}: {}", targetNode, replica);
Slice slice = null;
for (Slice s : coll.getSlices()) {

6
solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
View File

@ -102,8 +102,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
@ -143,9 +141,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
ZkStateReader.REPLICATION_FACTOR, "1",
ZkStateReader.NRT_REPLICAS, "1",
ZkStateReader.TLOG_REPLICAS, "0",
ZkStateReader.PULL_REPLICAS, "0",
WITH_COLLECTION, null,
COLOCATED_WITH, null));
ZkStateReader.PULL_REPLICAS, "0"));
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String FAILURE_FIELD = "failure";

2
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -136,7 +136,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES
import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME;
import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE;
import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
@ -463,7 +462,6 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
TLOG_REPLICAS,
NRT_REPLICAS,
WAIT_FOR_FINAL_STATE,
WITH_COLLECTION,
ALIAS);
if (props.get(REPLICATION_FACTOR) != null && props.get(NRT_REPLICAS) != null) {

2
solr/dev-docs/overseer/overseer.adoc
View File

@ -531,7 +531,7 @@ Then finally the real work is done! The (deprecated) http shard handler is used
It ends up executing `CoreAdminOperation.CREATE_OP` that calls `CoreContainer
.create()` for the core of the replica.
The rest of the method deals with failures, with `withCollection` and with `alias`.
The rest of the method deals with failures with `alias`.
[[bug9]]
[WARNING]

1
solr/solr-ref-guide/src/collection-management.adoc
View File

@ -225,7 +225,6 @@ The attributes that can be modified are:
* rule
* snitch
* policy
* withCollection
* readOnly
* other custom properties that use a `property.` prefix

2
solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
View File

@ -111,7 +111,7 @@ _(raw; not yet edited)_
* SOLR-14656: Autoscaling framework removed
This includes:
Autoscaling, policy, triggers etc.
withCollection handling
withCollection handling (SOLR-14964)
UTILIZENODE command
Sim framework
Suggestions tab in UI

5
solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
View File

@ -44,7 +44,6 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -74,7 +73,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
private Map<String, Object> snitchSession = new HashMap<>();
@SuppressWarnings({"rawtypes"})
private Map<String, Map> nodeVsTags = new HashMap<>();
private Map<String, String> withCollectionsMap = new HashMap<>();
public SolrClientNodeStateProvider(CloudSolrClient solrClient) {
this.solrClient = solrClient;
@ -100,9 +98,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
all.forEach((collName, ref) -> {
DocCollection coll = ref.get();
if (coll == null) return;
if (coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION) != null) {
withCollectionsMap.put(coll.getName(), (String) coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION));
}
coll.forEachReplica((shard, replica) -> {
Map<String, Map<String, List<Replica>>> nodeData = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());
Map<String, List<Replica>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());

4
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -61,13 +61,11 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX;
import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
/**
* This class is experimental and subject to change.
@ -82,8 +80,6 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public static final java.util.List<String> MODIFIABLE_COLLECTION_PROPERTIES = Arrays.asList(
REPLICATION_FACTOR,
COLL_CONF,
WITH_COLLECTION,
COLOCATED_WITH,
READ_ONLY);
protected final CollectionAction action;

11
solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
View File

@ -80,17 +80,6 @@ public interface CollectionAdminParams {
*/
String COLL_CONF = "collection.configName";
/**
* The name of the collection with which a collection is to be co-located
*/
String WITH_COLLECTION = "withCollection";
/**
* The reverse-link to WITH_COLLECTION flag. It is stored in the cluster state of the `withCollection`
* and points to the collection on which the `withCollection` was specified.
*/
String COLOCATED_WITH = "COLOCATED_WITH";
/**
* Used by cluster properties API as a wrapper key to provide defaults for collection, cluster etc.
*