Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13262: Add collection RENAME command and support using aliases in most collection admin commands.

This commit is contained in:
Andrzej Bialecki 2019-04-10 18:44:05 +02:00
parent eafe42f090
commit 02c4503f8c
36 changed files with 598 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -112,6 +112,8 @@ New Features
hierarchy and indexing the new one with the atomic update merged into it. Also, [child] Doc Transformer now works
with RealTimeGet. (Moshe Bla, David Smiley)
* SOLR-13262: Add collection RENAME command and support using aliases in most collection admin commands. (ab)
Bug Fixes
----------------------

4
solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
View File

@ -95,9 +95,11 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
throws IOException, InterruptedException {
log.debug("addReplica() : {}", Utils.toJSONString(message));
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String shard = message.getStr(SHARD_ID_PROP);
final String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
DocCollection coll = clusterState.getCollection(collectionName);
if (coll == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collectionName + " does not exist");

9
solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
View File

@ -67,7 +67,8 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
String backupName = message.getStr(NAME);
String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
@ -92,7 +93,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
switch (strategy) {
case CollectionAdminParams.COPY_FILES_STRATEGY: {
copyIndexFiles(backupPath, message, results);
copyIndexFiles(backupPath, collectionName, message, results);
break;
}
case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {
@ -115,6 +116,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);
properties.put(CollectionAdminParams.COLL_CONF, configName);
properties.put(BackupManager.START_TIME_PROP, startTime.toString());
properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
@ -155,8 +157,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
return r.get();
}
private void copyIndexFiles(URI backupPath, ZkNodeProps request, NamedList results) throws Exception {
String collectionName = request.getStr(COLLECTION_PROP);
private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, NamedList results) throws Exception {
String backupName = request.getStr(NAME);
String asyncId = request.getStr(ASYNC);
String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);

7
solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
View File

@ -31,6 +31,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
@ -43,7 +44,7 @@ public class CreateAliasCmd extends AliasCmd {
private final OverseerCollectionMessageHandler ocmh;
private static boolean anyRoutingParams(ZkNodeProps message) {
return message.keySet().stream().anyMatch(k -> k.startsWith(RoutedAlias.ROUTER_PREFIX));
return message.keySet().stream().anyMatch(k -> k.startsWith(CollectionAdminParams.ROUTER_PREFIX));
}
@SuppressWarnings("WeakerAccess")
@ -56,6 +57,10 @@ public class CreateAliasCmd extends AliasCmd {
throws Exception {
final String aliasName = message.getStr(CommonParams.NAME);
ZkStateReader zkStateReader = ocmh.zkStateReader;
// make sure we have the latest version of existing aliases
if (zkStateReader.aliasesManager != null) { // not a mock ZkStateReader
zkStateReader.aliasesManager.update();
}
if (!anyRoutingParams(message)) {
callCreatePlainAlias(message, aliasName, zkStateReader);

26
solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
View File

@ -45,6 +45,7 @@ import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
@ -80,6 +81,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@ -101,20 +103,29 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
if (ocmh.zkStateReader.aliasesManager != null) { // not a mock ZkStateReader
ocmh.zkStateReader.aliasesManager.update();
}
final Aliases aliases = ocmh.zkStateReader.getAliases();
final String collectionName = message.getStr(NAME);
final boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false);
final String alias = message.getStr(ALIAS, collectionName);
log.info("Create collection {}", collectionName);
if (clusterState.hasCollection(collectionName)) {
if (clusterState.hasCollection(collectionName) || aliases.hasAlias(collectionName)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
}
if (aliases.hasAlias(collectionName)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection alias already exists: " + collectionName);
}
String withCollection = message.getStr(CollectionAdminParams.WITH_COLLECTION);
String withCollectionShard = null;
if (withCollection != null) {
if (!clusterState.hasCollection(withCollection)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "The 'withCollection' does not exist: " + withCollection);
String realWithCollection = aliases.resolveSimpleAlias(withCollection);
if (!clusterState.hasCollection(realWithCollection)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "The 'withCollection' does not exist: " + realWithCollection);
} else {
DocCollection collection = clusterState.getCollection(withCollection);
DocCollection collection = clusterState.getCollection(realWithCollection);
if (collection.getActiveSlices().size() > 1) {
throw new SolrException(ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + collection.getActiveSlices().size());
}
@ -283,12 +294,14 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
ocmh.processResponses(results, shardHandler, false, null, async, requestMap, Collections.emptySet());
if(results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0) {
boolean failure = results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0;
if (failure) {
// Let's cleanup as we hit an exception
// We shouldn't be passing 'results' here for the cleanup as the response would then contain 'success'
// element, which may be interpreted by the user as a positive ack
ocmh.cleanupCollection(collectionName, new NamedList<Object>());
log.info("Cleaned up artifacts for failed create collection for [{}]", collectionName);
return;
} else {
log.debug("Finished create command on all shards for collection: {}", collectionName);
@ -318,6 +331,9 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
}
// create an alias pointing to the new collection
ocmh.zkStateReader.aliasesManager.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(alias, collectionName));
} catch (SolrException ex) {
throw ex;
} catch (Exception ex) {

5
solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
View File

@ -51,14 +51,15 @@ public class CreateShardCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String sliceName = message.getStr(SHARD_ID_PROP);
boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false);
log.info("Create shard invoked: {}", message);
if (collectionName == null || sliceName == null)
if (extCollectionName == null || sliceName == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters");
String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
DocCollection collection = clusterState.getCollection(collectionName);
int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));

4
solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java
View File

@ -64,7 +64,9 @@ public class CreateSnapshotCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String collectionName = ocmh.zkStateReader.getAliases().resolveSimpleAlias(extCollectionName);
String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
String asyncId = message.getStr(ASYNC);
SolrZkClient zkClient = ocmh.zkStateReader.getZkClient();

51
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
View File

@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.NonExistentCoreException;
@ -68,10 +69,18 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
final String collection = message.getStr(NAME);
final String extCollection = message.getStr(NAME);
ZkStateReader zkStateReader = ocmh.zkStateReader;
checkNotReferencedByAlias(zkStateReader, collection);
if (zkStateReader.aliasesManager != null) { // not a mock ZkStateReader
zkStateReader.aliasesManager.update(); // aliases may have been stale; get latest from ZK
}
String aliasReference = checkAliasReference(zkStateReader, extCollection);
Aliases aliases = zkStateReader.getAliases();
String collection = aliases.resolveSimpleAlias(extCollection);
checkNotColocatedWith(zkStateReader, collection);
final boolean deleteHistory = message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true);
@ -115,8 +124,8 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
okayExceptions.add(NonExistentCoreException.class.getName());
List<Replica> failedReplicas = ocmh.collectionCmd(message, params, results, null, asyncId, requestMap, okayExceptions);
for (Replica failedRepilca : failedReplicas) {
boolean isSharedFS = failedRepilca.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && failedRepilca.get("dataDir") != null;
for (Replica failedReplica : failedReplicas) {
boolean isSharedFS = failedReplica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && failedReplica.get("dataDir") != null;
if (isSharedFS) {
// if the replica use a shared FS and it did not receive the unload message, then counter node should not be removed
// because when a new collection with same name is created, new replicas may reuse the old dataDir
@ -130,7 +139,12 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
// wait for a while until we don't see the collection
zkStateReader.waitForState(collection, 60, TimeUnit.SECONDS, (liveNodes, collectionState) -> collectionState == null);
// we can delete any remaining unique alias
if (aliasReference != null) {
ocmh.zkStateReader.aliasesManager.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(aliasReference, null));
}
// TimeOut timeout = new TimeOut(60, TimeUnit.SECONDS, timeSource);
// boolean removed = false;
// while (! timeout.hasTimedOut()) {
@ -169,24 +183,33 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
}
private void checkNotReferencedByAlias(ZkStateReader zkStateReader, String collection) throws Exception {
String alias = referencedByAlias(collection, zkStateReader.getAliases());
if (alias != null) {
// it's ok if a collection is referenced either by none or exactly by a single alias.
// This method returns the single alias to delete, if present, or null
private String checkAliasReference(ZkStateReader zkStateReader, String extCollection) throws Exception {
List<String> aliases = referencedByAlias(extCollection, zkStateReader.getAliases());
if (aliases.size() > 1) {
zkStateReader.aliasesManager.update(); // aliases may have been stale; get latest from ZK
alias = referencedByAlias(collection, zkStateReader.getAliases());
if (alias != null) {
aliases = referencedByAlias(extCollection, zkStateReader.getAliases());
if (aliases.size() > 1) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Collection : " + collection + " is part of alias " + alias + " remove or modify the alias before removing this collection.");
"Collection : " + extCollection + " is part of aliases: " + aliases + ", remove or modify the aliases before removing this collection.");
}
}
if (!aliases.isEmpty()) {
return aliases.get(0);
} else {
return null;
}
}
public static String referencedByAlias(String collection, Aliases aliases) {
public static List<String> referencedByAlias(String extCollection, Aliases aliases) throws IllegalArgumentException {
Objects.requireNonNull(aliases);
// this quickly produces error if the name is a complex alias
String collection = aliases.resolveSimpleAlias(extCollection);
return aliases.getCollectionAliasListMap().entrySet().stream()
.filter(e -> e.getValue().contains(collection))
.filter(e -> e.getValue().contains(collection) || e.getValue().contains(extCollection))
.map(Map.Entry::getKey) // alias name
.findFirst().orElse(null);
.collect(Collectors.toList());
}
private void checkNotColocatedWith(ZkStateReader zkStateReader, String collection) throws Exception {

4
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
View File

@ -81,10 +81,12 @@ public class DeleteReplicaCmd implements Cmd {
ocmh.checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP);
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String shard = message.getStr(SHARD_ID_PROP);
String replicaName = message.getStr(REPLICA_PROP);
String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
DocCollection coll = clusterState.getCollection(collectionName);
Slice slice = coll.getSlice(shard);
if (slice == null) {

4
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java
View File

@ -62,9 +62,11 @@ public class DeleteShardCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
String extCollectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP);
String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
log.info("Delete shard invoked");
Slice slice = clusterState.getCollection(collectionName).getSlice(sliceId);
if (slice == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,

3
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java
View File

@ -64,7 +64,8 @@ public class DeleteSnapshotCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String extCollectionName = message.getStr(COLLECTION_PROP);
String collectionName = ocmh.zkStateReader.getAliases().resolveSimpleAlias(extCollectionName);
String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
String asyncId = message.getStr(ASYNC);
Map<String, String> requestMap = new HashMap<>();

7
solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
View File

@ -73,11 +73,14 @@ public class MigrateCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
String sourceCollectionName = message.getStr("collection");
String extSourceCollectionName = message.getStr("collection");
String splitKey = message.getStr("split.key");
String targetCollectionName = message.getStr("target.collection");
String extTargetCollectionName = message.getStr("target.collection");
int timeout = message.getInt("forward.timeout", 10 * 60) * 1000;
String sourceCollectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extSourceCollectionName);
String targetCollectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extTargetCollectionName);
DocCollection sourceCollection = clusterState.getCollection(sourceCollectionName);
if (sourceCollection == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName);

4
solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
View File

@ -72,7 +72,7 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
private void moveReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
log.debug("moveReplica() : {}", Utils.toJSONString(message));
ocmh.checkRequired(message, COLLECTION_PROP, CollectionParams.TARGET_NODE);
String collection = message.getStr(COLLECTION_PROP);
String extCollection = message.getStr(COLLECTION_PROP);
String targetNode = message.getStr(CollectionParams.TARGET_NODE);
boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false);
boolean inPlaceMove = message.getBool(IN_PLACE_MOVE, true);
@ -80,6 +80,8 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
String async = message.getStr(ASYNC);
String collection = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollection);
DocCollection coll = clusterState.getCollection(collection);
if (coll == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist");

17
solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
View File

@ -243,6 +243,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
.put(MOVEREPLICA, new MoveReplicaCmd(this))
.put(REINDEXCOLLECTION, new ReindexCollectionCmd(this))
.put(UTILIZENODE, new UtilizeNodeCmd(this))
.put(RENAME, new RenameCmd(this))
.build()
;
}
@ -455,6 +456,22 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
}
void checkResults(String label, NamedList<Object> results, boolean failureIsFatal) throws SolrException {
Object failure = results.get("failure");
if (failure == null) {
failure = results.get("error");
}
if (failure != null) {
String msg = "Error: " + label + ": " + Utils.toJSONString(results);
if (failureIsFatal) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
} else {
log.error(msg);
}
}
}
//TODO should we not remove in the next release ?
private void migrateStateFormat(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
final String collectionName = message.getStr(COLLECTION_PROP);

73
solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
View File

@ -44,7 +44,6 @@ import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
@ -174,32 +173,23 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
log.debug("*** called: {}", message);
String collection = message.getStr(CommonParams.NAME);
// before resolving aliases
String originalCollection = collection;
Aliases aliases = ocmh.zkStateReader.getAliases();
if (collection != null) {
// resolve aliases - the source may be an alias
List<String> aliasList = aliases.resolveAliases(collection);
if (aliasList != null && !aliasList.isEmpty()) {
collection = aliasList.get(0);
}
}
String extCollection = message.getStr(CommonParams.NAME);
if (collection == null || !clusterState.hasCollection(collection)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source collection name must be specified and must exist");
if (extCollection == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source collection name must be specified");
}
String collection = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollection);
if (!clusterState.hasCollection(collection)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source collection name must exist");
}
String target = message.getStr(TARGET);
if (target == null) {
target = collection;
} else {
// resolve aliases
List<String> aliasList = aliases.resolveAliases(target);
if (aliasList != null && !aliasList.isEmpty()) {
target = aliasList.get(0);
}
target = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(target);
}
boolean sameTarget = target.equals(collection) || target.equals(originalCollection);
boolean sameTarget = target.equals(collection) || target.equals(extCollection);
boolean removeSource = message.getBool(REMOVE_SOURCE, false);
Cmd command = Cmd.get(message.getStr(COMMAND, Cmd.START.toLower()));
if (command == null) {
@ -255,7 +245,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
int seq = tmpCollectionSeq.getAndIncrement();
if (sameTarget) {
do {
targetCollection = TARGET_COL_PREFIX + originalCollection + "_" + seq;
targetCollection = TARGET_COL_PREFIX + extCollection + "_" + seq;
if (!clusterState.hasCollection(targetCollection)) {
break;
}
@ -264,7 +254,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
} else {
targetCollection = target;
}
String chkCollection = CHK_COL_PREFIX + originalCollection;
String chkCollection = CHK_COL_PREFIX + extCollection;
String daemonUrl = null;
Exception exc = null;
boolean createdTarget = false;
@ -294,7 +284,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
checkResults("deleting old checkpoint collection " + chkCollection, cmdResults, true);
ocmh.checkResults("deleting old checkpoint collection " + chkCollection, cmdResults, true);
}
if (maybeAbort(collection)) {
@ -343,7 +333,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, cmd, cmdResults);
createdTarget = true;
checkResults("creating target collection " + targetCollection, cmdResults, true);
ocmh.checkResults("creating target collection " + targetCollection, cmdResults, true);
// create the checkpoint collection - use RF=1 and 1 shard
cmd = new ZkNodeProps(
@ -357,7 +347,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
);
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, cmd, cmdResults);
checkResults("creating checkpoint collection " + chkCollection, cmdResults, true);
ocmh.checkResults("creating checkpoint collection " + chkCollection, cmdResults, true);
// wait for a while until we see both collections
TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, ocmh.timeSource);
boolean created = false;
@ -439,14 +429,14 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
// 5. if (sameTarget) set up an alias to use targetCollection as the source name
if (sameTarget) {
log.debug("- setting up alias from " + originalCollection + " to " + targetCollection);
log.debug("- setting up alias from " + extCollection + " to " + targetCollection);
cmd = new ZkNodeProps(
CommonParams.NAME, originalCollection,
CommonParams.NAME, extCollection,
"collections", targetCollection);
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATEALIAS).call(clusterState, cmd, results);
checkResults("setting up alias " + originalCollection + " -> " + targetCollection, cmdResults, true);
reindexingState.put("alias", originalCollection + " -> " + targetCollection);
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATEALIAS).call(clusterState, cmd, cmdResults);
ocmh.checkResults("setting up alias " + extCollection + " -> " + targetCollection, cmdResults, true);
reindexingState.put("alias", extCollection + " -> " + targetCollection);
}
reindexingState.remove("daemonUrl");
@ -468,7 +458,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
);
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
checkResults("deleting checkpoint collection " + chkCollection, cmdResults, true);
ocmh.checkResults("deleting checkpoint collection " + chkCollection, cmdResults, true);
// 7. optionally delete the source collection
if (removeSource) {
@ -480,7 +470,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
);
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
checkResults("deleting source collection " + collection, cmdResults, true);
ocmh.checkResults("deleting source collection " + collection, cmdResults, true);
} else {
// 8. clear readOnly on source
ZkNodeProps props = new ZkNodeProps(
@ -500,7 +490,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
reindexingState.put(PHASE, "done");
removeReindexingState(collection);
} catch (Exception e) {
log.warn("Error during reindexing of " + originalCollection, e);
log.warn("Error during reindexing of " + extCollection, e);
exc = e;
aborted = true;
} finally {
@ -563,21 +553,6 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
}
private void checkResults(String label, NamedList<Object> results, boolean failureIsFatal) throws Exception {
Object failure = results.get("failure");
if (failure == null) {
failure = results.get("error");
}
if (failure != null) {
String msg = "Error: " + label + ": " + Utils.toJSONString(results);
if (failureIsFatal) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
} else {
log.error(msg);
}
}
}
private boolean maybeAbort(String collection) throws Exception {
DocCollection coll = ocmh.cloudManager.getClusterStateProvider().getClusterState().getCollectionOrNull(collection);
if (coll == null) {
@ -798,7 +773,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
checkResults("CLEANUP: deleting target collection " + targetCollection, cmdResults, false);
ocmh.checkResults("CLEANUP: deleting target collection " + targetCollection, cmdResults, false);
}
// remove chk collection
@ -811,7 +786,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
);
cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
checkResults("CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false);
ocmh.checkResults("CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false);
}
log.debug(" -- turning readOnly mode off for " + collection);
ZkNodeProps props = new ZkNodeProps(

70
solr/core/src/java/org/apache/solr/cloud/api/collections/RenameCmd.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud.api.collections;
import java.lang.invoke.MethodHandles;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
*/
public class RenameCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final OverseerCollectionMessageHandler ocmh;
public RenameCmd(OverseerCollectionMessageHandler ocmh) {
this.ocmh = ocmh;
}
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String extCollectionName = message.getStr(CoreAdminParams.NAME);
String target = message.getStr(CollectionAdminParams.TARGET);
if (ocmh.zkStateReader.aliasesManager != null) { // not a mock ZkStateReader
ocmh.zkStateReader.aliasesManager.update();
}
if (extCollectionName == null || target == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "both collection 'name' and 'target' name must be specified");
}
Aliases aliases = ocmh.zkStateReader.getAliases();
String collectionName = aliases.resolveSimpleAlias(extCollectionName);
if (!state.hasCollection(collectionName)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "source collection '" + collectionName + "' not found.");
}
if (ocmh.zkStateReader.getAliases().hasAlias(target)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "target alias '" + target + "' exists: "
+ ocmh.zkStateReader.getAliases().getCollectionAliasListMap().get(target));
}
ocmh.zkStateReader.aliasesManager.applyModificationAndExportToZk(a -> a.cloneWithRename(extCollectionName, target));
}
}

7
solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
View File

@ -107,6 +107,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
Properties properties = backupMgr.readBackupProperties(location, backupName);
String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP);
String backupCollectionAlias = properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP);
DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection);
// Get the Solr nodes to restore a collection.
@ -416,6 +417,12 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
}
}
if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) {
log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection);
ocmh.zkStateReader.aliasesManager
.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection));
}
log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName);
} finally {
if (sessionWrapper != null) sessionWrapper.release();

2
solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
View File

@ -29,6 +29,7 @@ import org.apache.solr.update.AddUpdateCommand;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX;
public interface RoutedAlias {
@ -40,7 +41,6 @@ public interface RoutedAlias {
CATEGORY
}
String ROUTER_PREFIX = "router.";
String ROUTER_TYPE_NAME = ROUTER_PREFIX + "name";
String ROUTER_FIELD = ROUTER_PREFIX + "field";
String CREATE_COLLECTION_PREFIX = "create-collection.";

4
solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
View File

@ -107,7 +107,9 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
}
boolean withTiming = message.getBool(CommonParams.TIMING, false);
String collectionName = message.getStr(CoreAdminParams.COLLECTION);
String extCollectionName = message.getStr(CoreAdminParams.COLLECTION);
String collectionName = ocmh.cloudManager.getClusterStateProvider().resolveSimpleAlias(extCollectionName);
log.debug("Split shard invoked: {}", message);
ZkStateReader zkStateReader = ocmh.zkStateReader;

1
solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
View File

@ -61,6 +61,7 @@ import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType
import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.NONE;
import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.SYNCHRONOUS;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX;
import static org.apache.solr.common.params.CommonParams.TZ;
/**

1
solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
View File

@ -62,6 +62,7 @@ public class BackupManager {
// Backup properties
public static final String COLLECTION_NAME_PROP = "collection";
public static final String COLLECTION_ALIAS_PROP = "collectionAlias";
public static final String BACKUP_NAME_PROP = "backupName";
public static final String INDEX_VERSION_PROP = "index.version";
public static final String START_TIME_PROP = "startTime";

35
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -137,6 +137,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
@ -481,7 +482,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
NRT_REPLICAS,
POLICY,
WAIT_FOR_FINAL_STATE,
WITH_COLLECTION);
WITH_COLLECTION,
ALIAS);
props.putIfAbsent(STATE_FORMAT, "2");
@ -542,6 +544,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
RELOAD_OP(RELOAD, (req, rsp, h) -> copy(req.getParams().required(), null, NAME)),
RENAME_OP(RENAME, (req, rsp, h) -> copy(req.getParams().required(), null, NAME, CollectionAdminParams.TARGET)),
REINDEXCOLLECTION_OP(REINDEXCOLLECTION, (req, rsp, h) -> {
Map<String, Object> m = copy(req.getParams().required(), null, NAME);
copy(req.getParams(), m,
@ -572,7 +576,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}),
SYNCSHARD_OP(SYNCSHARD, (req, rsp, h) -> {
String collection = req.getParams().required().get("collection");
String extCollection = req.getParams().required().get("collection");
String collection = h.coreContainer.getZkController().getZkStateReader().getAliases().resolveSimpleAlias(extCollection);
String shard = req.getParams().required().get("shard");
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
@ -811,7 +816,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return null;
}),
COLLECTIONPROP_OP(COLLECTIONPROP, (req, rsp, h) -> {
String collection = req.getParams().required().get(NAME);
String extCollection = req.getParams().required().get(NAME);
String collection = h.coreContainer.getZkController().getZkStateReader().getAliases().resolveSimpleAlias(extCollection);
String name = req.getParams().required().get(PROPERTY_NAME);
String val = req.getParams().get(PROPERTY_VALUE);
CollectionProperties cp = new CollectionProperties(h.coreContainer.getZkController().getZkClient());
@ -921,6 +927,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
NamedList<Object> results = new NamedList<>();
Map<String, DocCollection> collections = h.coreContainer.getZkController().getZkStateReader().getClusterState().getCollectionsMap();
List<String> collectionList = new ArrayList<>(collections.keySet());
// XXX should we add aliases here?
results.add("collections", collectionList);
SolrResponse response = new OverseerSolrResponse(results);
rsp.getValues().addAll(response.getResponse());
@ -1027,7 +1034,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
BACKUP_OP(BACKUP, (req, rsp, h) -> {
req.getParams().required().check(NAME, COLLECTION_PROP);
String collectionName = req.getParams().get(COLLECTION_PROP);
String extCollectionName = req.getParams().get(COLLECTION_PROP);
String collectionName = h.coreContainer.getZkController().getZkStateReader()
.getAliases().resolveSimpleAlias(extCollectionName);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
@ -1077,6 +1086,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
if (clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' exists, no action taken.");
}
if (h.coreContainer.getZkController().getZkStateReader().getAliases().hasAlias(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' is an existing alias, no action taken.");
}
CoreContainer cc = h.coreContainer;
String repo = req.getParams().get(CoreAdminParams.BACKUP_REPOSITORY);
@ -1126,7 +1138,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
CREATESNAPSHOT_OP(CREATESNAPSHOT, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
String collectionName = req.getParams().get(COLLECTION_PROP);
String extCollectionName = req.getParams().get(COLLECTION_PROP);
String collectionName = h.coreContainer.getZkController().getZkStateReader()
.getAliases().resolveSimpleAlias(extCollectionName);
String commitName = req.getParams().get(CoreAdminParams.COMMIT_NAME);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
@ -1146,7 +1160,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
DELETESNAPSHOT_OP(DELETESNAPSHOT, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
String collectionName = req.getParams().get(COLLECTION_PROP);
String extCollectionName = req.getParams().get(COLLECTION_PROP);
String collectionName = h.coreContainer.getZkController().getZkStateReader()
.getAliases().resolveSimpleAlias(extCollectionName);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
@ -1158,7 +1174,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
LISTSNAPSHOTS_OP(LISTSNAPSHOTS, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP);
String collectionName = req.getParams().get(COLLECTION_PROP);
String extCollectionName = req.getParams().get(COLLECTION_PROP);
String collectionName = h.coreContainer.getZkController().getZkStateReader()
.getAliases().resolveSimpleAlias(extCollectionName);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
@ -1256,7 +1274,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
private static void forceLeaderElection(SolrQueryRequest req, CollectionsHandler handler) {
ZkController zkController = handler.coreContainer.getZkController();
ClusterState clusterState = zkController.getClusterState();
String collectionName = req.getParams().required().get(COLLECTION_PROP);
String extCollectionName = req.getParams().required().get(COLLECTION_PROP);
String collectionName = zkController.zkStateReader.getAliases().resolveSimpleAlias(extCollectionName);
String sliceId = req.getParams().required().get(SHARD_ID_PROP);
log.info("Force leader invoked, state: {}", clusterState);

9
solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
View File

@ -22,6 +22,7 @@ import java.util.EnumSet;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@ -60,13 +61,17 @@ class SolrSchema extends AbstractSchema {
final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
for (String collection : clusterState.getCollectionsMap().keySet()) {
Set<String> collections = clusterState.getCollectionsMap().keySet();
for (String collection : collections) {
builder.put(collection, new SolrTable(this, collection));
}
Aliases aliases = zkStateReader.getAliases();
for (String alias : aliases.getCollectionAliasListMap().keySet()) {
builder.put(alias, new SolrTable(this, alias));
// don't create duplicate entries
if (!collections.contains(alias)) {
builder.put(alias, new SolrTable(this, alias));
}
}
return builder.build();

10
solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
View File

@ -17,7 +17,6 @@
package org.apache.solr.search.join;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import org.apache.lucene.index.DocValuesType;
@ -294,14 +293,13 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private static String resolveAlias(String fromIndex, ZkController zkController) {
final Aliases aliases = zkController.getZkStateReader().getAliases();
List<String> collections = aliases.resolveAliases(fromIndex); // if not an alias, returns input
if (collections.size() != 1) {
try {
return aliases.resolveSimpleAlias(fromIndex); // if not an alias, returns input
} catch (IllegalArgumentException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: Collection alias '" + fromIndex +
"' maps to multiple collections (" + collections +
"), which is not currently supported for joins.");
"' maps to multiple collectiions, which is not currently supported for joins.", e);
}
return collections.get(0);
}
private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {

57
solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
View File

@ -54,6 +54,7 @@ import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@ -791,6 +792,62 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals("num docs after turning off read-only", NUM_DOCS * 3, rsp.getResults().getNumFound());
}
@Test
public void testRenameCollection() throws Exception {
String collectionName1 = "testRename_collection1";
String collectionName2 = "testRename_collection2";
CollectionAdminRequest.createCollection(collectionName1, "conf", 1, 1).setAlias("col1").process(cluster.getSolrClient());
CollectionAdminRequest.createCollection(collectionName2, "conf", 1, 1).setAlias("col2").process(cluster.getSolrClient());
cluster.waitForActiveCollection(collectionName1, 1, 1);
cluster.waitForActiveCollection(collectionName2, 1, 1);
waitForState("Expected collection1 to be created with 1 shard and 1 replica", collectionName1, clusterShape(1, 1));
waitForState("Expected collection2 to be created with 1 shard and 1 replica", collectionName2, clusterShape(1, 1));
CollectionAdminRequest.createAlias("compoundAlias", "col1,col2").process(cluster.getSolrClient());
CollectionAdminRequest.createAlias("simpleAlias", "col1").process(cluster.getSolrClient());
CollectionAdminRequest.createCategoryRoutedAlias("catAlias", "field1", 100,
CollectionAdminRequest.createCollection("_unused_", "conf", 1, 1)).process(cluster.getSolrClient());
CollectionAdminRequest.renameCollection("col1", "foo").process(cluster.getSolrClient());
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
zkStateReader.aliasesManager.update();
Aliases aliases = zkStateReader.getAliases();
assertEquals(aliases.getCollectionAliasListMap().toString(), collectionName1, aliases.resolveSimpleAlias("foo"));
assertEquals(aliases.getCollectionAliasListMap().toString(), collectionName1, aliases.resolveSimpleAlias("simpleAlias"));
List<String> compoundAliases = aliases.resolveAliases("compoundAlias");
assertEquals(compoundAliases.toString(), 2, compoundAliases.size());
assertTrue(compoundAliases.toString(), compoundAliases.contains(collectionName1));
assertTrue(compoundAliases.toString(), compoundAliases.contains(collectionName2));
CollectionAdminRequest.renameCollection(collectionName1, collectionName2).process(cluster.getSolrClient());
zkStateReader.aliasesManager.update();
aliases = zkStateReader.getAliases();
assertEquals(aliases.getCollectionAliasListMap().toString(), collectionName2, aliases.resolveSimpleAlias("foo"));
assertEquals(aliases.getCollectionAliasListMap().toString(), collectionName2, aliases.resolveSimpleAlias("simpleAlias"));
assertEquals(aliases.getCollectionAliasListMap().toString(), collectionName2, aliases.resolveSimpleAlias(collectionName1));
// we renamed col1 -> col2 so the compound alias contains only "col2,col2" which is reduced to col2
compoundAliases = aliases.resolveAliases("compoundAlias");
assertEquals(compoundAliases.toString(), 1, compoundAliases.size());
assertTrue(compoundAliases.toString(), compoundAliases.contains(collectionName2));
try {
CollectionAdminRequest.renameCollection("catAlias", "bar").process(cluster.getSolrClient());
fail("category-based alias renaming should fail");
} catch (Exception e) {
assertTrue(e.toString().contains("is a routed alias"));
}
try {
CollectionAdminRequest.renameCollection("col2", "foo").process(cluster.getSolrClient());
fail("shuold fail because 'foo' already exists");
} catch (Exception e) {
assertTrue(e.toString().contains("exists"));
}
}
@Test
public void testOverseerStatus() throws IOException, SolrServerException {

11
solr/solr-ref-guide/src/aliases.adoc
View File

@ -48,6 +48,17 @@ With multiple collections in an alias this is always a problem, so if you have a
However, for analytical use cases where results are sorted on numeric, date or alphanumeric field values rather
than relevancy calculations this is not a problem.
== Collection admin commands and aliases
Starting with version 8.1 SolrCloud supports using alias names in collection admin commands where normally a
collection name is expected. This works only when the following criteria are satisfied:
* an alias must not refer to more than one collection
* an alias must not refer to a Routed Alias (see below)
If all criteria are satisfied then the command will resolve alias names and operate on the collections the aliases
refer to, as if it was invoked with the collection names instead. Otherwise the command will not be executed and
an exception will be thrown.
== Routed Aliases
To address the update limitations associated with standard aliases and provide additional useful features, the concept of

64
solr/solr-ref-guide/src/collections-api.adoc
View File

@ -120,6 +120,11 @@ If `true`, the request will complete only when all affected replicas become acti
The name of the collection with which all replicas of this collection must be co-located. The collection must already exist and must have a single shard named `shard1`.
See <<colocating-collections.adoc#colocating-collections, Colocating collections>> for more details.
`alias`::
Starting with version 8.1 when a collection is created additionally an alias (by default with the same name) is created
that points to this collection. This parameter allows changing the name of this alias, effectively combining
this operation with <<createalias, CREATEALIAS>>
Collections are first created in read-write mode but can be put in `readOnly`
mode using the <<modifycollection, MODIFYCOLLECTION>> action.
@ -392,6 +397,65 @@ http://localhost:8983/solr/admin/collections?action=RELOAD&name=newCollection&wt
</response>
----
[[rename]]
== RENAME: Rename a Collection
`/admin/collections?action=RENAME&name=_existingName_&target=_targetName_`
Renaming a collection sets up a standard alias that points to the underlying collection, so
that the same (unmodified) collection can now be referred to in query, index and admin operations
using the new name.
This command does NOT actually rename the underlying Solr collection - it sets up a new one-to-one alias
using the new name, or renames the existing alias so that it uses the new name, while still referring to
the same underlying Solr collection. However, from the user's point of view the collection can now be
accessed using the new name, and the new name can be also referred to in other aliases.
The following limitations apply:
* the existing name must be either a SolrCloud collection or a standard alias referring to a single collection.
Aliases that refer to more than 1 collection are not supported.
* the existing name must not be a Routed Alias.
* the target name must not be an existing alias.
=== RENAME Command Parameters
`name`::
Name of the existing SolrCloud collection or an alias that refers to exactly one collection and is not
a Routed Alias.
`target`::
Target name of the collection. This will be the new alias that refers to the underlying SolrCloud collection.
The original name (or alias) of the collection will be replaced also in the existing aliases so that they
also refer to the new name. Target name must not be an existing alias.
=== Examples using RENAME
Assuming there are two actual SolrCloud collections named `collection1` and `collection2`,
and the following aliases already exist:
* `col1 -&gt; collection1`: this resolves to `collection1`.
* `col2 -&gt; collection2`: this resolves to `collection2`.
* `simpleAlias -&gt; col1`: this resolves to `collection1`.
* `compoundAlias -&gt; col1,col2`: this resolves to `collection1,collection2`
The RENAME of `col1` to `foo` will change the aliases to the following:
* `foo -&gt; collection1`: this resolves to `collection1`.
* `col2 -&gt; collection2`: this resolves to `collection2`.
* `simpleAlias -&gt; foo`: this resolves to `collection1`.
* `compoundAlias -&gt; foo,col2`: this resolves to `collection1,collection2`.
If we then rename `collection1` (which is an actual collection name) to `collection2` (which is also
an actual collection name) the following aliases will exist now:
* `foo -&gt; collection2`: this resolves to `collection2`.
* `col2 -&gt; collection2`: this resolves to `collection2`.
* `simpleAlias -&gt; foo`: this resolves to `collection2`.
* `compoundAlias -&gt; foo,col2`: this would resolve now to `collection2,collection2` so it's reduced to simply `collection2`.
* `collection1` -&gt; `collection2`: this newly created alias effectively hides `collection1` from regular query and
update commands, which are directed now to `collection2`.
[[splitshard]]
== SPLITSHARD: Split a Shard

9
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
View File

@ -63,6 +63,15 @@ public class DelegatingClusterStateProvider implements ClusterStateProvider {
}
}
@Override
public String resolveSimpleAlias(String alias) throws IllegalArgumentException {
if (delegate != null) {
return delegate.resolveSimpleAlias(alias);
} else {
return alias;
}
}
@Override
public ClusterState getClusterState() throws IOException {
if (delegate != null) {

6
solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseCloudSolrClient.java
View File

@ -1062,11 +1062,7 @@ public abstract class BaseCloudSolrClient extends SolrClient {
for (String collectionName : inputCollections) {
if (getClusterStateProvider().getState(collectionName) == null) {
// perhaps it's an alias
List<String> aliasedCollections = getClusterStateProvider().resolveAlias(collectionName);
// one more level of alias indirection... (dubious that we should support this)
for (String aliasedCollection : aliasedCollections) {
collectionNames.addAll(getClusterStateProvider().resolveAlias(aliasedCollection));
}
collectionNames.addAll(getClusterStateProvider().resolveAlias(collectionName));
} else {
collectionNames.add(collectionName); // it's a collection
}

5
solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
View File

@ -192,6 +192,11 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid
return Aliases.resolveAliasesGivenAliasMap(getAliases(false), aliasName);
}
@Override
public String resolveSimpleAlias(String aliasName) throws IllegalArgumentException {
return Aliases.resolveSimpleAliasGivenAliasMap(getAliases(false), aliasName);
}
private Map<String, List<String>> getAliases(boolean forceFetch) {
if (this.liveNodes == null) {
throw new RuntimeException("We don't know of any live_nodes to fetch the"

13
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
View File

@ -44,6 +44,19 @@ public interface ClusterStateProvider extends SolrCloseable {
*/
List<String> resolveAlias(String alias);
/**
* Given a collection alias, return a single collection it points to, or the original name if it's not an
* alias.
* @throws IllegalArgumentException if an alias points to more than 1 collection, either directly or indirectly.
*/
default String resolveSimpleAlias(String alias) throws IllegalArgumentException {
List<String> aliases = resolveAlias(alias);
if (aliases.size() > 1) {
throw new IllegalArgumentException("Simple alias '" + alias + "' points to more than 1 collection: " + aliases);
}
return aliases.get(0);
}
/**
* Obtain the current cluster state.
*/

5
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java
View File

@ -92,6 +92,11 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {
return zkStateReader.getAliases().resolveAliases(alias); // if not an alias, returns itself
}
@Override
public String resolveSimpleAlias(String alias) throws IllegalArgumentException {
return zkStateReader.getAliases().resolveSimpleAlias(alias);
}
@Override
public Object getClusterProperty(String propertyName) {
Map<String, Object> props = zkStateReader.getClusterProperties();

28
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -67,6 +67,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
/**
@ -433,6 +434,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected Properties properties;
protected Boolean autoAddReplicas;
protected String alias;
protected Integer stateFormat;
protected String[] rule , snitch;
protected String withCollection;
@ -476,6 +478,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public Create setRule(String... s){ this.rule = s; return this; }
public Create setSnitch(String... s){ this.snitch = s; return this; }
public Create setAlias(String alias) {
this.alias = alias;
return this;
}
public String getConfigName() { return configName; }
public String getCreateNodeSet() { return createNodeSet; }
public String getRouterName() { return routerName; }
@ -573,6 +580,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (snitch != null) params.set(DocCollection.SNITCH, snitch);
params.setNonNull(POLICY, policy);
params.setNonNull(WITH_COLLECTION, withCollection);
params.setNonNull(ALIAS, alias);
return params;
}
@ -606,6 +614,26 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
public static Rename renameCollection(String collection, String target) {
return new Rename(collection, target);
}
public static class Rename extends AsyncCollectionSpecificAdminRequest {
String target;
public Rename(String collection, String target) {
super(CollectionAction.RENAME, collection);
this.target = target;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CollectionAdminParams.TARGET, target);
return params;
}
}
/**
* Returns a SolrRequest to delete a node.
*/

138
solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.common.cloud;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -26,6 +27,7 @@ import java.util.Objects;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
@ -180,26 +182,74 @@ public class Aliases {
return resolveAliasesGivenAliasMap(collectionAliases, aliasName);
}
/**
* Returns true if an alias is defined, false otherwise.
*/
public boolean hasAlias(String aliasName) {
return collectionAliases.containsKey(aliasName);
}
/**
* Resolve an alias that points to a single collection. One level of alias indirection is supported.
* @param aliasName alias name
* @return original name if there's no such alias, or a resolved name. If an alias points to more than 1
* collection (directly or indirectly) an exception is thrown
* @throws IllegalArgumentException if either direct or indirect alias points to more than 1 name.
*/
public String resolveSimpleAlias(String aliasName) throws IllegalArgumentException {
return resolveSimpleAliasGivenAliasMap(collectionAliases, aliasName);
}
/** @lucene.internal */
@SuppressWarnings("JavaDoc")
public static String resolveSimpleAliasGivenAliasMap(Map<String, List<String>> collectionAliasListMap,
String aliasName) throws IllegalArgumentException {
List<String> level1 = collectionAliasListMap.get(aliasName);
if (level1 == null || level1.isEmpty()) {
return aliasName; // simple collection name
}
if (level1.size() > 1) {
throw new IllegalArgumentException("Simple alias '" + aliasName + "' points to more than 1 collection: " + level1);
}
List<String> level2 = collectionAliasListMap.get(level1.get(0));
if (level2 == null || level2.isEmpty()) {
return level1.get(0); // simple alias
}
if (level2.size() > 1) {
throw new IllegalArgumentException("Simple alias '" + aliasName + "' resolves to '"
+ level1.get(0) + "' which points to more than 1 collection: " + level2);
}
return level2.get(0);
}
/** @lucene.internal */
@SuppressWarnings("JavaDoc")
public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
//return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
// TODO deprecate and remove this dubious feature?
// Due to another level of indirection, this is more complicated...
List<String> level1 = collectionAliasListMap.get(aliasName);
if (level1 == null) {
return Collections.singletonList(aliasName);// is a collection
}
List<String> result = new ArrayList<>(level1.size());
for (String level1Alias : level1) {
// avoid allocating objects if possible
LinkedHashSet<String> uniqueResult = null;
for (int i = 0; i < level1.size(); i++) {
String level1Alias = level1.get(i);
List<String> level2 = collectionAliasListMap.get(level1Alias);
if (level2 == null) {
result.add(level1Alias);
if (level2 == null && uniqueResult != null) {
uniqueResult.add(level1Alias);
} else {
result.addAll(level2);
if (uniqueResult == null) { // lazy init
uniqueResult = new LinkedHashSet<>(level1.size());
uniqueResult.addAll(level1.subList(0, i));
}
uniqueResult.addAll(level2);
}
}
return Collections.unmodifiableList(result);
if (uniqueResult == null) {
return level1;
} else {
return Collections.unmodifiableList(new ArrayList<>(uniqueResult));
}
}
/**
@ -222,6 +272,15 @@ public class Aliases {
newColProperties = new LinkedHashMap<>(this.collectionAliasProperties);//clone to modify
newColProperties.remove(alias);
newColAliases.remove(alias);
// remove second-level alias from compound aliases
for (Map.Entry<String, List<String>> entry : newColAliases.entrySet()) {
List<String> list = entry.getValue();
if (list.contains(alias)) {
list = new ArrayList<>(list);
list.remove(alias);
entry.setValue(Collections.unmodifiableList(list));
}
}
} else {
newColProperties = this.collectionAliasProperties;// no changes
// java representation is a list, so split before adding to maintain consistency
@ -230,6 +289,69 @@ public class Aliases {
return new Aliases(newColAliases, newColProperties, zNodeVersion);
}
/**
* Rename an alias. This performs a "deep rename", which changes also the second-level alias lists.
* Renaming routed aliases is not supported.
* <p>
* Note that the state in zookeeper is unaffected by this method and the change must still be persisted via
* {@link ZkStateReader.AliasesManager#applyModificationAndExportToZk(UnaryOperator)}
*
* @param before previous alias name, must not be null
* @param after new alias name. If this is null then it's equivalent to calling {@link #cloneWithCollectionAlias(String, String)}
* with the second argument set to null, ie. removing an alias.
* @return new instance with the renamed alias
* @throws IllegalArgumentException when either <code>before</code> or <code>after</code> is empty, or
* the <code>before</code> name is a routed alias
*/
public Aliases cloneWithRename(String before, String after) {
if (before == null) {
throw new NullPointerException("'before' and 'after' cannot be null");
}
if (after == null) {
return cloneWithCollectionAlias(before, after);
}
if (before.isEmpty() || after.isEmpty()) {
throw new IllegalArgumentException("'before' and 'after' cannot be empty");
}
if (before.equals(after)) {
return this;
}
Map<String, String> props = collectionAliasProperties.get(before);
if (props != null) {
if (props.keySet().stream().anyMatch(k -> k.startsWith(CollectionAdminParams.ROUTER_PREFIX))) {
throw new IllegalArgumentException("source name '" + before + "' is a routed alias.");
}
}
Map<String, Map<String, String>> newColProperties = new LinkedHashMap<>(this.collectionAliasProperties);
Map<String, List<String>> newColAliases = new LinkedHashMap<>(this.collectionAliases);//clone to modify
List<String> level1 = newColAliases.remove(before);
props = newColProperties.remove(before);
if (level1 != null) {
newColAliases.put(after, level1);
}
if (props != null) {
newColProperties.put(after, props);
}
for (Map.Entry<String, List<String>> entry : newColAliases.entrySet()) {
List<String> collections = entry.getValue();
if (collections.contains(before)) {
LinkedHashSet<String> newCollections = new LinkedHashSet<>(collections.size());
for (String coll : collections) {
if (coll.equals(before)) {
newCollections.add(after);
} else {
newCollections.add(coll);
}
}
entry.setValue(Collections.unmodifiableList(new ArrayList<>(newCollections)));
}
}
if (level1 == null) { // create an alias that points to the collection
newColAliases.put(before, Collections.singletonList(after));
}
return new Aliases(newColAliases, newColProperties, zNodeVersion);
}
/**
* Set the value for some properties on a collection alias. This is done by creating a new Aliases instance
* with the same data as the current one but with a modification based on the parameters.

15
solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
View File

@ -109,4 +109,19 @@ public interface CollectionAdminParams {
* or the autoscaling policy based strategy to assign replicas to nodes. The default is false.
*/
String USE_LEGACY_REPLICA_ASSIGNMENT = "useLegacyReplicaAssignment";
/**
* When creating a collection create also a specified alias.
*/
String ALIAS = "alias";
/**
* Specifies the target of RENAME operation.
*/
String TARGET = "target";
/**
* Prefix for {@link org.apache.solr.common.cloud.DocRouter} properties
*/
String ROUTER_PREFIX = "router.";
}

3
solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
View File

@ -125,7 +125,8 @@ public interface CollectionParams {
MERGESHARDS(true, LockLevel.SHARD),
COLSTATUS(true, LockLevel.NONE),
// this command implements its own locking
REINDEXCOLLECTION(true, LockLevel.NONE)
REINDEXCOLLECTION(true, LockLevel.NONE),
RENAME(true, LockLevel.COLLECTION)
;
public final boolean isWrite;