Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-09 11:35:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

SOLR-9326: Ability to create/delete/list snapshots at collection level.

Browse Source
This commit is contained in:
yonik 2016-10-21 07:08:47 -04:00
parent f379dde2d7
commit 3ce7552aa5
14 changed files with 1309 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -109,6 +109,9 @@ New Features
* SOLR-8370: Display configured Similarity in Schema-Browser, both global/default and per-field/field-type
(janhoy, Alexandre Rafalovitch)
* SOLR-9326: Ability to create/delete/list snapshots at collection level.
(Hrishikesh Gadre via yonik)
Bug Fixes
----------------------

75
solr/core/src/java/org/apache/solr/cloud/BackupCmd.java
View File

@ -19,16 +19,21 @@ package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -36,6 +41,10 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.backup.BackupManager;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,6 +72,21 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
String asyncId = message.getStr(ASYNC);
String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
if (commitName != null) {
SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
if (!snapshotMeta.isPresent()) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
+ " does not exist for collection " + collectionName);
}
if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
+ " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
}
}
Map<String, String> requestMap = new HashMap<>();
Instant startTime = Instant.now();
@ -85,8 +109,28 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
backupPath);
Collection<String> shardsToConsider = Collections.emptySet();
if (snapshotMeta.isPresent()) {
shardsToConsider = snapshotMeta.get().getShards();
}
for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
Replica replica = slice.getLeader();
Replica replica = null;
if (snapshotMeta.isPresent()) {
if (!shardsToConsider.contains(slice.getName())) {
log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
slice.getName(), collectionName, snapshotMeta.get().getName());
continue;
}
replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
} else {
// Note - Actually this can return a null value when there is no leader for this shard.
replica = slice.getLeader();
if (replica == null) {
throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
}
}
String coreName = replica.getStr(CORE_NAME_PROP);
@ -96,6 +140,9 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
params.set(CORE_NAME_PROP, coreName);
if (snapshotMeta.isPresent()) {
params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
}
ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
@ -129,4 +176,30 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
log.info("Completed backing up ZK data for backupName={}", backupName);
}
private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMeta, Slice slice) {
// The goal here is to choose the snapshot of the replica which was the leader at the time snapshot was created.
// If that is not possible, we choose any other replica for the given shard.
Collection<CoreSnapshotMetaData> snapshots = snapshotMeta.getReplicaSnapshotsForShard(slice.getName());
Optional<CoreSnapshotMetaData> leaderCore = snapshots.stream().filter(x -> x.isLeader()).findFirst();
if (leaderCore.isPresent()) {
log.info("Replica {} was the leader when snapshot {} was created.", leaderCore.get().getCoreName(), snapshotMeta.getName());
Replica r = slice.getReplica(leaderCore.get().getCoreName());
if ((r != null) && !r.getState().equals(State.DOWN)) {
return r;
}
}
Optional<Replica> r = slice.getReplicas().stream()
.filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x))
.findFirst();
if (!r.isPresent()) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName());
}
return r.get();
}
}

179
solr/core/src/java/org/apache/solr/cloud/CreateSnapshotCmd.java Normal file
View File

@ -0,0 +1,179 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class implements the functionality of creating a collection level snapshot.
*/
public class CreateSnapshotCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final OverseerCollectionMessageHandler ocmh;
public CreateSnapshotCmd (OverseerCollectionMessageHandler ocmh) {
this.ocmh = ocmh;
}
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
String asyncId = message.getStr(ASYNC);
SolrZkClient zkClient = this.ocmh.overseer.getZkController().getZkClient();
Date creationDate = new Date();
if(SolrSnapshotManager.snapshotExists(zkClient, collectionName, commitName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
+ " already exists for collection " + collectionName);
}
log.info("Creating a snapshot for collection={} with commitName={}", collectionName, commitName);
// Create a node in ZK to store the collection level snapshot meta-data.
SolrSnapshotManager.createCollectionLevelSnapshot(zkClient, collectionName, new CollectionSnapshotMetaData(commitName));
log.info("Created a ZK path to store snapshot information for collection={} with commitName={}", collectionName, commitName);
Map<String, String> requestMap = new HashMap<>();
NamedList shardRequestResults = new NamedList();
Map<String, Slice> shardByCoreName = new HashMap<>();
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getSlices()) {
for (Replica replica : slice.getReplicas()) {
if (replica.getState() != State.ACTIVE) {
log.info("Replica {} is not active. Hence not sending the createsnapshot request", replica.getCoreName());
continue; // Since replica is not active - no point sending a request.
}
String coreName = replica.getStr(CORE_NAME_PROP);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATESNAPSHOT.toString());
params.set(NAME, slice.getName());
params.set(CORE_NAME_PROP, coreName);
params.set(CoreAdminParams.COMMIT_NAME, commitName);
ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
log.debug("Sent createsnapshot request to core={} with commitName={}", coreName, commitName);
shardByCoreName.put(coreName, slice);
}
}
// At this point we want to make sure that at-least one replica for every shard
// is able to create the snapshot. If that is not the case, then we fail the request.
// This is to take care of the situation where e.g. entire shard is unavailable.
Set<String> failedShards = new HashSet<>();
ocmh.processResponses(shardRequestResults, shardHandler, false, null, asyncId, requestMap);
NamedList success = (NamedList) shardRequestResults.get("success");
List<CoreSnapshotMetaData> replicas = new ArrayList<>();
if (success != null) {
for ( int i = 0 ; i < success.size() ; i++) {
NamedList resp = (NamedList)success.getVal(i);
// Check if this core is the leader for the shard. The idea here is that during the backup
// operation we preferably use the snapshot of the "leader" replica since it is most likely
// to have latest state.
String coreName = (String)resp.get(CoreAdminParams.CORE);
Slice slice = shardByCoreName.remove(coreName);
boolean leader = (slice.getLeader() != null && slice.getLeader().getCoreName().equals(coreName));
resp.add(SolrSnapshotManager.SHARD_ID, slice.getName());
resp.add(SolrSnapshotManager.LEADER, leader);
CoreSnapshotMetaData c = new CoreSnapshotMetaData(resp);
replicas.add(c);
log.info("Snapshot with commitName {} is created successfully for core {}", commitName, c.getCoreName());
}
}
if (!shardByCoreName.isEmpty()) { // One or more failures.
log.warn("Unable to create a snapshot with name {} for following cores {}", commitName, shardByCoreName.keySet());
// Count number of failures per shard.
Map<String, Integer> failuresByShardId = new HashMap<>();
for (Map.Entry<String,Slice> entry : shardByCoreName.entrySet()) {
int f = 0;
if (failuresByShardId.get(entry.getValue().getName()) != null) {
f = failuresByShardId.get(entry.getValue().getName());
}
failuresByShardId.put(entry.getValue().getName(), f + 1);
}
// Now that we know number of failures per shard, we can figure out
// if at-least one replica per shard was able to create a snapshot or not.
DocCollection collectionStatus = ocmh.zkStateReader.getClusterState().getCollection(collectionName);
for (Map.Entry<String,Integer> entry : failuresByShardId.entrySet()) {
int replicaCount = collectionStatus.getSlice(entry.getKey()).getReplicas().size();
if (replicaCount <= entry.getValue()) {
failedShards.add(entry.getKey());
}
}
}
if (failedShards.isEmpty()) { // No failures.
CollectionSnapshotMetaData meta = new CollectionSnapshotMetaData(commitName, SnapshotStatus.Successful, creationDate, replicas);
SolrSnapshotManager.updateCollectionLevelSnapshot(zkClient, collectionName, meta);
log.info("Saved following snapshot information for collection={} with commitName={} in Zookeeper : {}", collectionName,
commitName, meta.toNamedList());
} else {
log.warn("Failed to create a snapshot for collection {} with commitName = {}. Snapshot could not be captured for following shards {}",
collectionName, commitName, failedShards);
// Update the ZK meta-data to include only cores with the snapshot. This will enable users to figure out
// which cores have the named snapshot.
CollectionSnapshotMetaData meta = new CollectionSnapshotMetaData(commitName, SnapshotStatus.Failed, creationDate, replicas);
SolrSnapshotManager.updateCollectionLevelSnapshot(zkClient, collectionName, meta);
log.info("Saved following snapshot information for collection={} with commitName={} in Zookeeper : {}", collectionName,
commitName, meta.toNamedList());
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to create snapshot on shards " + failedShards);
}
}
}

160
solr/core/src/java/org/apache/solr/cloud/DeleteSnapshotCmd.java Normal file
View File

@ -0,0 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class implements the functionality of deleting a collection level snapshot.
*/
public class DeleteSnapshotCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final OverseerCollectionMessageHandler ocmh;
public DeleteSnapshotCmd (OverseerCollectionMessageHandler ocmh) {
this.ocmh = ocmh;
}
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
String asyncId = message.getStr(ASYNC);
Map<String, String> requestMap = new HashMap<>();
NamedList shardRequestResults = new NamedList();
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
Optional<CollectionSnapshotMetaData> meta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
if (!meta.isPresent()) { // Snapshot not found. Nothing to do.
return;
}
log.info("Deleting a snapshot for collection={} with commitName={}", collectionName, commitName);
Set<String> existingCores = new HashSet<>();
for (Slice s : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getSlices()) {
for (Replica r : s.getReplicas()) {
existingCores.add(r.getCoreName());
}
}
Set<String> coresWithSnapshot = new HashSet<>();
for (CoreSnapshotMetaData m : meta.get().getReplicaSnapshots()) {
if (existingCores.contains(m.getCoreName())) {
coresWithSnapshot.add(m.getCoreName());
}
}
log.info("Existing cores with snapshot for collection={} are {}", collectionName, existingCores);
for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getSlices()) {
for (Replica replica : slice.getReplicas()) {
if (replica.getState() == State.DOWN) {
continue; // Since replica is down - no point sending a request.
}
// Note - when a snapshot is found in_progress state - it is the result of overseer
// failure while handling the snapshot creation. Since we don't know the exact set of
// replicas to contact at this point, we try on all replicas.
if (meta.get().getStatus() == SnapshotStatus.InProgress || coresWithSnapshot.contains(replica.getCoreName())) {
String coreName = replica.getStr(CORE_NAME_PROP);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.DELETESNAPSHOT.toString());
params.set(NAME, slice.getName());
params.set(CORE_NAME_PROP, coreName);
params.set(CoreAdminParams.COMMIT_NAME, commitName);
log.info("Sending deletesnapshot request to core={} with commitName={}", coreName, commitName);
ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
}
}
}
ocmh.processResponses(shardRequestResults, shardHandler, false, null, asyncId, requestMap);
NamedList success = (NamedList) shardRequestResults.get("success");
List<CoreSnapshotMetaData> replicas = new ArrayList<>();
if (success != null) {
for ( int i = 0 ; i < success.size() ; i++) {
NamedList resp = (NamedList)success.getVal(i);
// Unfortunately async processing logic doesn't provide the "core" name automatically.
String coreName = (String)resp.get("core");
coresWithSnapshot.remove(coreName);
}
}
if (!coresWithSnapshot.isEmpty()) { // One or more failures.
log.warn("Failed to delete a snapshot for collection {} with commitName = {}. Snapshot could not be deleted for following cores {}",
collectionName, commitName, coresWithSnapshot);
List<CoreSnapshotMetaData> replicasWithSnapshot = new ArrayList<>();
for (CoreSnapshotMetaData m : meta.get().getReplicaSnapshots()) {
if (coresWithSnapshot.contains(m.getCoreName())) {
replicasWithSnapshot.add(m);
}
}
// Update the ZK meta-data to include only cores with the snapshot. This will enable users to figure out
// which cores still contain the named snapshot.
CollectionSnapshotMetaData newResult = new CollectionSnapshotMetaData(meta.get().getName(), SnapshotStatus.Failed,
meta.get().getCreationDate(), replicasWithSnapshot);
SolrSnapshotManager.updateCollectionLevelSnapshot(zkClient, collectionName, newResult);
log.info("Saved snapshot information for collection={} with commitName={} in Zookeeper as follows", collectionName, commitName,
Utils.toJSON(newResult));
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to delete snapshot on cores " + coresWithSnapshot);
} else {
// Delete the ZK path so that we eliminate the references of this snapshot from collection level meta-data.
SolrSnapshotManager.deleteCollectionLevelSnapshot(zkClient, collectionName, commitName);
log.info("Deleted Zookeeper snapshot metdata for collection={} with commitName={}", collectionName, commitName);
log.info("Successfully deleted snapshot for collection={} with commitName={}", collectionName, commitName);
}
}
}

2
solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
View File

@ -182,6 +182,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
.put(DELETENODE, new DeleteNodeCmd(this))
.put(BACKUP, new BackupCmd(this))
.put(RESTORE, new RestoreCmd(this))
.put(CREATESNAPSHOT, new CreateSnapshotCmd(this))
.put(DELETESNAPSHOT, new DeleteSnapshotCmd(this))
.put(SPLITSHARD, new SplitShardCmd(this))
.put(ADDROLE, new OverseerRoleCmd(this, ADDROLE, overseerPrioritizer))
.put(REMOVEROLE, new OverseerRoleCmd(this, REMOVEROLE, overseerPrioritizer))

242
solr/core/src/java/org/apache/solr/core/snapshots/CollectionSnapshotMetaData.java Normal file
View File

@ -0,0 +1,242 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.noggit.JSONWriter;
/**
* This class defines the meta-data about a collection level snapshot
*/
public class CollectionSnapshotMetaData implements JSONWriter.Writable {
public static class CoreSnapshotMetaData implements JSONWriter.Writable {
private final String coreName;
private final String indexDirPath;
private final long generationNumber;
private final boolean leader;
private final String shardId;
private final Collection<String> files;
public CoreSnapshotMetaData(String coreName, String indexDirPath, long generationNumber, String shardId, boolean leader, Collection<String> files) {
this.coreName = coreName;
this.indexDirPath = indexDirPath;
this.generationNumber = generationNumber;
this.shardId = shardId;
this.leader = leader;
this.files = files;
}
@SuppressWarnings({"unchecked", "rawtypes"})
public CoreSnapshotMetaData(NamedList resp) {
this.coreName = (String)resp.get(CoreAdminParams.CORE);
this.indexDirPath = (String)resp.get(SolrSnapshotManager.INDEX_DIR_PATH);
this.generationNumber = (Long)resp.get(SolrSnapshotManager.GENERATION_NUM);
this.shardId = (String)resp.get(SolrSnapshotManager.SHARD_ID);
this.leader = (Boolean)resp.get(SolrSnapshotManager.LEADER);
this.files = (Collection<String>)resp.get(SolrSnapshotManager.FILE_LIST);
}
public String getCoreName() {
return coreName;
}
public String getIndexDirPath() {
return indexDirPath;
}
public long getGenerationNumber() {
return generationNumber;
}
public Collection<String> getFiles() {
return files;
}
public String getShardId() {
return shardId;
}
public boolean isLeader() {
return leader;
}
@Override
public void write(JSONWriter arg0) {
LinkedHashMap<String, Object> info = new LinkedHashMap<String, Object>();
info.put(CoreAdminParams.CORE, getCoreName());
info.put(SolrSnapshotManager.INDEX_DIR_PATH, getIndexDirPath());
info.put(SolrSnapshotManager.GENERATION_NUM, getGenerationNumber());
info.put(SolrSnapshotManager.SHARD_ID, getShardId());
info.put(SolrSnapshotManager.LEADER, isLeader());
info.put(SolrSnapshotManager.FILE_LIST, getFiles());
arg0.write(info);
}
@SuppressWarnings({"rawtypes", "unchecked"})
public NamedList toNamedList() {
NamedList result = new NamedList();
result.add(CoreAdminParams.CORE, getCoreName());
result.add(SolrSnapshotManager.INDEX_DIR_PATH, getIndexDirPath());
result.add(SolrSnapshotManager.GENERATION_NUM, getGenerationNumber());
result.add(SolrSnapshotManager.SHARD_ID, getShardId());
result.add(SolrSnapshotManager.LEADER, isLeader());
result.add(SolrSnapshotManager.FILE_LIST, getFiles());
return result;
}
}
public static enum SnapshotStatus {
Successful, InProgress, Failed;
}
private final String name;
private final SnapshotStatus status;
private final Date creationDate;
private final List<CoreSnapshotMetaData> replicaSnapshots;
public CollectionSnapshotMetaData(String name) {
this(name, SnapshotStatus.InProgress, new Date(), Collections.<CoreSnapshotMetaData>emptyList());
}
public CollectionSnapshotMetaData(String name, SnapshotStatus status, Date creationTime, List<CoreSnapshotMetaData> replicaSnapshots) {
this.name = name;
this.status = status;
this.creationDate = creationTime;
this.replicaSnapshots = replicaSnapshots;
}
@SuppressWarnings("unchecked")
public CollectionSnapshotMetaData(Map<String, Object> data) {
this.name = (String)data.get(CoreAdminParams.NAME);
this.status = SnapshotStatus.valueOf((String)data.get(SolrSnapshotManager.SNAPSHOT_STATUS));
this.creationDate = new Date((Long)data.get(SolrSnapshotManager.CREATION_DATE));
this.replicaSnapshots = new ArrayList<>();
List<Object> r = (List<Object>) data.get(SolrSnapshotManager.SNAPSHOT_REPLICAS);
for (Object x : r) {
Map<String, Object> info = (Map<String, Object>)x;
String coreName = (String)info.get(CoreAdminParams.CORE);
String indexDirPath = (String)info.get(SolrSnapshotManager.INDEX_DIR_PATH);
long generationNumber = (Long) info.get(SolrSnapshotManager.GENERATION_NUM);
String shardId = (String)info.get(SolrSnapshotManager.SHARD_ID);
boolean leader = (Boolean) info.get(SolrSnapshotManager.LEADER);
Collection<String> files = (Collection<String>)info.get(SolrSnapshotManager.FILE_LIST);
replicaSnapshots.add(new CoreSnapshotMetaData(coreName, indexDirPath, generationNumber, shardId, leader, files));
}
}
@SuppressWarnings("unchecked")
public CollectionSnapshotMetaData(NamedList<Object> data) {
this.name = (String)data.get(CoreAdminParams.NAME);
String statusStr = (String)data.get(SolrSnapshotManager.SNAPSHOT_STATUS);
this.creationDate = new Date((Long)data.get(SolrSnapshotManager.CREATION_DATE));
this.status = SnapshotStatus.valueOf(statusStr);
this.replicaSnapshots = new ArrayList<>();
NamedList<Object> r = (NamedList<Object>) data.get(SolrSnapshotManager.SNAPSHOT_REPLICAS);
for (Map.Entry<String,Object> x : r) {
NamedList<Object> info = (NamedList<Object>)x.getValue();
String coreName = (String)info.get(CoreAdminParams.CORE);
String indexDirPath = (String)info.get(SolrSnapshotManager.INDEX_DIR_PATH);
long generationNumber = (Long) info.get(SolrSnapshotManager.GENERATION_NUM);
String shardId = (String)info.get(SolrSnapshotManager.SHARD_ID);
boolean leader = (Boolean) info.get(SolrSnapshotManager.LEADER);
Collection<String> files = (Collection<String>)info.get(SolrSnapshotManager.FILE_LIST);
replicaSnapshots.add(new CoreSnapshotMetaData(coreName, indexDirPath, generationNumber, shardId, leader, files));
}
}
public String getName() {
return name;
}
public SnapshotStatus getStatus() {
return status;
}
public Date getCreationDate() {
return creationDate;
}
public List<CoreSnapshotMetaData> getReplicaSnapshots() {
return replicaSnapshots;
}
public List<CoreSnapshotMetaData> getReplicaSnapshotsForShard(String shardId) {
List<CoreSnapshotMetaData> result = new ArrayList<>();
for (CoreSnapshotMetaData d : replicaSnapshots) {
if (d.getShardId().equals(shardId)) {
result.add(d);
}
}
return result;
}
public boolean isSnapshotExists(String shardId, Replica r) {
for (CoreSnapshotMetaData d : replicaSnapshots) {
if (d.getShardId().equals(shardId) && d.getCoreName().equals(r.getCoreName())) {
return true;
}
}
return false;
}
public Collection<String> getShards() {
Set<String> result = new HashSet<>();
for (CoreSnapshotMetaData d : replicaSnapshots) {
result.add(d.getShardId());
}
return result;
}
@Override
public void write(JSONWriter arg0) {
LinkedHashMap<String, Object> result = new LinkedHashMap<>();
result.put(CoreAdminParams.NAME, this.name);
result.put(SolrSnapshotManager.SNAPSHOT_STATUS, this.status.toString());
result.put(SolrSnapshotManager.CREATION_DATE, this.getCreationDate().getTime());
result.put(SolrSnapshotManager.SNAPSHOT_REPLICAS, this.replicaSnapshots);
arg0.write(result);
}
@SuppressWarnings({"rawtypes", "unchecked"})
public NamedList toNamedList() {
NamedList result = new NamedList();
result.add(CoreAdminParams.NAME, this.name);
result.add(SolrSnapshotManager.SNAPSHOT_STATUS, this.status.toString());
result.add(SolrSnapshotManager.CREATION_DATE, this.getCreationDate().getTime());
NamedList replicas = new NamedList();
for (CoreSnapshotMetaData x : replicaSnapshots) {
replicas.add(x.getCoreName(), x.toNamedList());
}
result.add(SolrSnapshotManager.SNAPSHOT_REPLICAS, replicas);
return result;
}
}

180
solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java
View File

@ -18,9 +18,12 @@ package org.apache.solr.core.snapshots;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
@ -28,9 +31,13 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -40,6 +47,172 @@ import org.slf4j.LoggerFactory;
public class SolrSnapshotManager {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String INDEX_DIR_PATH = "indexDirPath";
public static final String GENERATION_NUM = "generation";
public static final String SNAPSHOT_STATUS = "status";
public static final String CREATION_DATE = "creationDate";
public static final String SNAPSHOT_REPLICAS = "replicas";
public static final String SNAPSHOTS_INFO = "snapshots";
public static final String LEADER = "leader";
public static final String SHARD_ID = "shard_id";
public static final String FILE_LIST = "files";
/**
* This method returns if a named snapshot exists for the specified collection.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @param commitName The name of the snapshot
* @return true if the named snapshot exists
* false Otherwise
* @throws KeeperException In case of Zookeeper error
* @throws InterruptedException In case of thread interruption.
*/
public static boolean snapshotExists(SolrZkClient zkClient, String collectionName, String commitName)
throws KeeperException, InterruptedException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.ofNullable(commitName));
return zkClient.exists(zkPath, true);
}
/**
* This method creates an entry for the named snapshot for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @param meta The {@linkplain CollectionSnapshotMetaData} corresponding to named snapshot
* @throws KeeperException In case of Zookeeper error
* @throws InterruptedException In case of thread interruption.
*/
public static void createCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName,
CollectionSnapshotMetaData meta) throws KeeperException, InterruptedException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(meta.getName()));
zkClient.makePath(zkPath, Utils.toJSON(meta), CreateMode.PERSISTENT, true);
}
/**
* This method updates an entry for the named snapshot for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @param meta The {@linkplain CollectionSnapshotMetaData} corresponding to named snapshot
* @throws KeeperException In case of Zookeeper error
* @throws InterruptedException In case of thread interruption.
*/
public static void updateCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName,
CollectionSnapshotMetaData meta) throws KeeperException, InterruptedException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(meta.getName()));
zkClient.setData(zkPath, Utils.toJSON(meta), -1, true);
}
/**
* This method deletes an entry for the named snapshot for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @param commitName The name of the snapshot
* @throws InterruptedException In case of thread interruption.
* @throws KeeperException In case of Zookeeper error
*/
public static void deleteCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, String commitName)
throws InterruptedException, KeeperException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(commitName));
zkClient.delete(zkPath, -1, true);
}
/**
* This method deletes all snapshots for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @throws InterruptedException In case of thread interruption.
* @throws KeeperException In case of Zookeeper error
*/
public static void cleanupCollectionLevelSnapshots(SolrZkClient zkClient, String collectionName)
throws InterruptedException, KeeperException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.empty());
try {
// Delete the meta-data for each snapshot.
Collection<String> snapshots = zkClient.getChildren(zkPath, null, true);
for (String snapshot : snapshots) {
String path = getSnapshotMetaDataZkPath(collectionName, Optional.of(snapshot));
try {
zkClient.delete(path, -1, true);
} catch (KeeperException ex) {
// Gracefully handle the case when the zk node doesn't exist
if ( ex.code() != KeeperException.Code.NONODE ) {
throw ex;
}
}
}
// Delete the parent node.
zkClient.delete(zkPath, -1, true);
} catch (KeeperException ex) {
// Gracefully handle the case when the zk node doesn't exist (e.g. if no snapshots were created for this collection).
if ( ex.code() != KeeperException.Code.NONODE ) {
throw ex;
}
}
}
/**
* This method returns the {@linkplain CollectionSnapshotMetaData} for the named snapshot for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @param commitName The name of the snapshot
* @return (Optional) the {@linkplain CollectionSnapshotMetaData}
* @throws InterruptedException In case of thread interruption.
* @throws KeeperException In case of Zookeeper error
*/
public static Optional<CollectionSnapshotMetaData> getCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, String commitName)
throws InterruptedException, KeeperException {
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(commitName));
try {
Map<String, Object> data = (Map<String, Object>)Utils.fromJSON(zkClient.getData(zkPath, null, null, true));
return Optional.of(new CollectionSnapshotMetaData(data));
} catch (KeeperException ex) {
// Gracefully handle the case when the zk node for a specific
// snapshot doesn't exist (e.g. due to a concurrent delete operation).
if ( ex.code() == KeeperException.Code.NONODE ) {
return Optional.empty();
}
throw ex;
}
}
/**
* This method returns the {@linkplain CollectionSnapshotMetaData} for each named snapshot for the specified collection in Zookeeper.
*
* @param zkClient Zookeeper client
* @param collectionName The name of the collection
* @return the {@linkplain CollectionSnapshotMetaData} for each named snapshot
* @throws InterruptedException In case of thread interruption.
* @throws KeeperException In case of Zookeeper error
*/
public static Collection<CollectionSnapshotMetaData> listSnapshots(SolrZkClient zkClient, String collectionName)
throws InterruptedException, KeeperException {
Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.empty());
try {
Collection<String> snapshots = zkClient.getChildren(zkPath, null, true);
for (String snapshot : snapshots) {
Optional<CollectionSnapshotMetaData> s = getCollectionLevelSnapshot(zkClient, collectionName, snapshot);
if (s.isPresent()) {
result.add(s.get());
}
}
} catch (KeeperException ex) {
// Gracefully handle the case when the zk node doesn't exist (e.g. due to a concurrent delete collection operation).
if ( ex.code() != KeeperException.Code.NONODE ) {
throw ex;
}
}
return result;
}
/**
* This method deletes index files of the {@linkplain IndexCommit} for the specified generation number.
*
@ -117,4 +290,11 @@ public class SolrSnapshotManager {
// Note the index writer creates a new commit during the close() operation (which is harmless).
}
}
private static String getSnapshotMetaDataZkPath(String collectionName, Optional<String> commitName) {
if (commitName.isPresent()) {
return "/snapshots/"+collectionName+"/"+commitName.get();
}
return "/snapshots/"+collectionName;
}
}

54
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -77,6 +77,8 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.request.SolrQueryRequest;
@ -732,7 +734,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex);
}
Map<String, Object> params = req.getParams().getAll(null, NAME, COLLECTION_PROP);
Map<String, Object> params = req.getParams().getAll(null, NAME, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
params.put(CoreAdminParams.BACKUP_LOCATION, location);
return params;
}),
@ -778,7 +780,57 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
copyPropertiesWithPrefix(req.getParams(), params, COLL_PROP_PREFIX);
return params;
}),
CREATESNAPSHOT_OP(CREATESNAPSHOT, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
String collectionName = req.getParams().get(COLLECTION_PROP);
String commitName = req.getParams().get(CoreAdminParams.COMMIT_NAME);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
}
SolrZkClient client = h.coreContainer.getZkController().getZkClient();
if (SolrSnapshotManager.snapshotExists(client, collectionName, commitName)) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Snapshot with name '" + commitName + "' already exists for collection '"
+ collectionName + "', no action taken.");
}
Map<String, Object> params = req.getParams().getAll(null, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
return params;
}),
DELETESNAPSHOT_OP(DELETESNAPSHOT, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
String collectionName = req.getParams().get(COLLECTION_PROP);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
}
Map<String, Object> params = req.getParams().getAll(null, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
return params;
}),
LISTSNAPSHOTS_OP(LISTSNAPSHOTS, (req, rsp, h) -> {
req.getParams().required().check(COLLECTION_PROP);
String collectionName = req.getParams().get(COLLECTION_PROP);
ClusterState clusterState = h.coreContainer.getZkController().getClusterState();
if (!clusterState.hasCollection(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
}
NamedList<Object> snapshots = new NamedList<Object>();
SolrZkClient client = h.coreContainer.getZkController().getZkClient();
Collection<CollectionSnapshotMetaData> m = SolrSnapshotManager.listSnapshots(client, collectionName);
for (CollectionSnapshotMetaData meta : m) {
snapshots.add(meta.getName(), meta.toNamedList());
}
rsp.add(SolrSnapshotManager.SNAPSHOTS_INFO, snapshots);
return null;
}),
REPLACENODE_OP(REPLACENODE, (req, rsp, h) -> req.getParams().required().getAll(req.getParams().getAll(null, "parallel"), "source", "target")),
DELETENODE_OP(DELETENODE, (req, rsp, h) -> req.getParams().required().getAll(null, "node"));
public final CollectionOp fun;

7
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
View File

@ -37,6 +37,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp;
@ -270,12 +271,12 @@ enum CoreAdminOperation implements CoreAdminOp {
Optional<SnapshotMetaData> metadata = mgr.getSnapshotMetaData(name);
if ( metadata.isPresent() ) {
NamedList<String> props = new NamedList<>();
props.add("generation", String.valueOf(metadata.get().getGenerationNumber()));
props.add("indexDirPath", metadata.get().getIndexDirPath());
props.add(SolrSnapshotManager.GENERATION_NUM, String.valueOf(metadata.get().getGenerationNumber()));
props.add(SolrSnapshotManager.INDEX_DIR_PATH, metadata.get().getIndexDirPath());
result.add(name, props);
}
}
it.rsp.add("snapshots", result);
it.rsp.add(SolrSnapshotManager.SNAPSHOTS_INFO, result);
}
});

10
solr/core/src/java/org/apache/solr/handler/admin/CreateSnapshotOp.java
View File

@ -23,6 +23,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RefCounted;
@ -53,10 +54,11 @@ class CreateSnapshotOp implements CoreAdminHandler.CoreAdminOp {
SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager();
mgr.snapshot(commitName, indexDirPath, ic.getGeneration());
it.rsp.add("core", core.getName());
it.rsp.add("commitName", commitName);
it.rsp.add("indexDirPath", indexDirPath);
it.rsp.add("generation", ic.getGeneration());
it.rsp.add(CoreAdminParams.CORE, core.getName());
it.rsp.add(CoreAdminParams.COMMIT_NAME, commitName);
it.rsp.add(SolrSnapshotManager.INDEX_DIR_PATH, indexDirPath);
it.rsp.add(SolrSnapshotManager.GENERATION_NUM, ic.getGeneration());
it.rsp.add(SolrSnapshotManager.FILE_LIST, ic.getFileNames());
}
}
}

4
solr/core/src/java/org/apache/solr/handler/admin/DeleteSnapshotOp.java
View File

@ -40,6 +40,10 @@ class DeleteSnapshotOp implements CoreAdminHandler.CoreAdminOp {
try {
core.deleteNamedSnapshot(commitName);
// Ideally we shouldn't need this. This is added since the RPC logic in
// OverseerCollectionMessageHandler can not provide the coreName as part of the result.
it.rsp.add(CoreAdminParams.CORE, core.getName());
it.rsp.add(CoreAdminParams.COMMIT_NAME, commitName);
} finally {
core.close();
}

285
solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java Normal file
View File

@ -0,0 +1,285 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.BackupRestoreUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
@Slow
public class TestSolrCloudSnapshots extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static long docsSeed; // see indexDocs()
private static final int NUM_SHARDS = 2;
private static final int NUM_REPLICAS = 2;
private static final int NUM_NODES = NUM_REPLICAS * NUM_SHARDS;
@BeforeClass
public static void setupClass() throws Exception {
useFactory("solr.StandardDirectoryFactory");
configureCluster(NUM_NODES)// nodes
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
docsSeed = random().nextLong();
}
@AfterClass
public static void teardownClass() throws Exception {
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
}
@Test
public void testSnapshots() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String collectionName = "SolrCloudSnapshots";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", NUM_SHARDS, NUM_REPLICAS);
create.process(solrClient);
int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
BackupRestoreUtils.verifyDocs(nDocs, solrClient, collectionName);
String commitName = TestUtil.randomSimpleString(random(), 1, 5);
// Verify if snapshot creation works with replica failures.
boolean replicaFailures = usually();
Optional<String> stoppedCoreName = Optional.empty();
if (replicaFailures) {
// Here the assumption is that Solr will spread the replicas uniformly across nodes.
// If this is not true for some reason, then we will need to add some logic to find a
// node with a single replica.
this.cluster.getRandomJetty(random()).stop();
// Sleep a bit for allowing ZK watch to fire.
Thread.sleep(5000);
// Figure out if at-least one replica is "down".
DocCollection collState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for (Slice s : collState.getSlices()) {
for (Replica replica : s.getReplicas()) {
if (replica.getState() == State.DOWN) {
stoppedCoreName = Optional.of(replica.getCoreName());
}
}
}
}
int expectedCoresWithSnapshot = stoppedCoreName.isPresent() ? (NUM_SHARDS * NUM_REPLICAS) - 1 : (NUM_SHARDS * NUM_REPLICAS);
CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, commitName);
createSnap.process(solrClient);
Collection<CollectionSnapshotMetaData> collectionSnaps = listCollectionSnapshots(solrClient, collectionName);
assertEquals(1, collectionSnaps.size());
CollectionSnapshotMetaData meta = collectionSnaps.iterator().next();
assertEquals(commitName, meta.getName());
assertEquals(CollectionSnapshotMetaData.SnapshotStatus.Successful, meta.getStatus());
assertEquals(expectedCoresWithSnapshot, meta.getReplicaSnapshots().size());
Map<String, CoreSnapshotMetaData> snapshotByCoreName = meta.getReplicaSnapshots().stream()
.collect(Collectors.toMap(CoreSnapshotMetaData::getCoreName, Function.identity()));
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(2, collectionState.getActiveSlices().size());
for ( Slice shard : collectionState.getActiveSlices() ) {
assertEquals(2, shard.getReplicas().size());
for (Replica replica : shard.getReplicas()) {
if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
continue; // We know that the snapshot is not created for this replica.
}
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
assertTrue(snapshotByCoreName.containsKey(coreName));
CoreSnapshotMetaData coreSnapshot = snapshotByCoreName.get(coreName);
try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
assertTrue("Snapshot not created for core " + coreName, metaData.isPresent());
assertEquals(coreSnapshot.getIndexDirPath(), metaData.get().getIndexDirPath());
assertEquals(coreSnapshot.getGenerationNumber(), metaData.get().getGenerationNumber());
}
}
}
// Delete all documents.
{
solrClient.deleteByQuery(collectionName, "*:*");
solrClient.commit(collectionName);
BackupRestoreUtils.verifyDocs(0, solrClient, collectionName);
}
String backupLocation = createTempDir().toFile().getAbsolutePath();
String backupName = "mytestbackup";
String restoreCollectionName = collectionName + "_restored";
//Create a backup using the earlier created snapshot.
{
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
.setLocation(backupLocation).setCommitName(commitName);
if (random().nextBoolean()) {
assertEquals(0, backup.process(solrClient).getStatus());
} else {
assertEquals(RequestStatusState.COMPLETED, backup.processAndWait(solrClient, 30));//async
}
}
// Restore backup.
{
CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
.setLocation(backupLocation);
if (random().nextBoolean()) {
assertEquals(0, restore.process(solrClient).getStatus());
} else {
assertEquals(RequestStatusState.COMPLETED, restore.processAndWait(solrClient, 30));//async
}
AbstractDistribZkTestBase.waitForRecoveriesToFinish(
restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
BackupRestoreUtils.verifyDocs(nDocs, solrClient, restoreCollectionName);
}
// Verify if the snapshot deletion works correctly when one or more replicas containing the snapshot are
// deleted
boolean replicaDeletion = rarely();
if (replicaDeletion) {
CoreSnapshotMetaData replicaToDelete = null;
for (String shardId : meta.getShards()) {
List<CoreSnapshotMetaData> replicas = meta.getReplicaSnapshotsForShard(shardId);
if (replicas.size() > 1) {
int r_index = random().nextInt(replicas.size());
replicaToDelete = replicas.get(r_index);
}
}
if (replicaToDelete != null) {
collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for (Slice s : collectionState.getSlices()) {
for (Replica r : s.getReplicas()) {
if (r.getCoreName().equals(replicaToDelete.getCoreName())) {
log.info("Deleting replica {}", r);
CollectionAdminRequest.DeleteReplica delReplica = CollectionAdminRequest.deleteReplica(collectionName,
replicaToDelete.getShardId(), r.getName());
delReplica.process(solrClient);
// The replica deletion will cleanup the snapshot meta-data.
snapshotByCoreName.remove(r.getCoreName());
break;
}
}
}
}
}
// Delete snapshot
CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, commitName);
deleteSnap.process(solrClient);
// Wait for a while so that the clusterstate.json updates are propagated to the client side.
Thread.sleep(2000);
collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for ( Slice shard : collectionState.getActiveSlices() ) {
for (Replica replica : shard.getReplicas()) {
if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
continue; // We know that the snapshot was not created for this replica.
}
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
assertFalse("Snapshot not deleted for core " + coreName, metaData.isPresent());
// Remove the entry for core if the snapshot is deleted successfully.
snapshotByCoreName.remove(coreName);
}
}
}
// Verify all core-level snapshots are deleted.
assertTrue("The cores remaining " + snapshotByCoreName, snapshotByCoreName.isEmpty());
assertTrue(listCollectionSnapshots(solrClient, collectionName).isEmpty());
}
private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(SolrClient adminClient, String collectionName) throws Exception {
CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots(collectionName);
CollectionAdminResponse resp = listSnapshots.process(adminClient);
assertTrue( resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO) instanceof NamedList );
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
for (int i = 0; i < apiResult.size(); i++) {
result.add(new CollectionSnapshotMetaData((NamedList<Object>)apiResult.getVal(i)));
}
return result;
}
private Collection<SnapshotMetaData> listCoreSnapshots(SolrClient adminClient, String coreName) throws Exception {
ListSnapshots req = new ListSnapshots();
req.setCoreName(coreName);
NamedList resp = adminClient.request(req);
assertTrue( resp.get(SolrSnapshotManager.SNAPSHOTS_INFO) instanceof NamedList );
NamedList apiResult = (NamedList) resp.get(SolrSnapshotManager.SNAPSHOTS_INFO);
List<SnapshotMetaData> result = new ArrayList<>(apiResult.size());
for(int i = 0 ; i < apiResult.size(); i++) {
String commitName = apiResult.getName(i);
String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.INDEX_DIR_PATH);
long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get(SolrSnapshotManager.GENERATION_NUM));
result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
}
return result;
}
}

116
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -706,8 +706,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
// BACKUP request
public static class Backup extends AsyncCollectionSpecificAdminRequest {
protected final String name;
protected Optional<String> repositoryName;
protected Optional<String> repositoryName = Optional.empty();
protected String location;
protected Optional<String> commitName = Optional.empty();
public Backup(String collection, String name) {
super(CollectionAction.BACKUP, collection);
@ -746,6 +747,15 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
public Optional<String> getCommitName() {
return commitName;
}
public Backup setCommitName(String commitName) {
this.commitName = Optional.ofNullable(commitName);
return this;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
@ -755,6 +765,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (repositoryName.isPresent()) {
params.set(CoreAdminParams.BACKUP_REPOSITORY, repositoryName.get());
}
if (commitName.isPresent()) {
params.set(CoreAdminParams.COMMIT_NAME, commitName.get());
}
return params;
}
@ -767,7 +780,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
// RESTORE request
public static class Restore extends AsyncCollectionSpecificAdminRequest {
protected final String backupName;
protected Optional<String> repositoryName;
protected Optional<String> repositoryName = Optional.empty();
protected String location;
// in common with collection creation:
@ -860,6 +873,105 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
//Note : This method is added since solrj module does not use Google
// guava library. Also changes committed for SOLR-8765 result in wrong
// error message when "collection" parameter is specified as Null.
// This is because the setCollectionName method is deprecated.
static <T> T checkNotNull(String param, T value) {
if (value == null) {
throw new NullPointerException("Please specify a value for parameter " + param);
}
return value;
}
@SuppressWarnings("serial")
public static class CreateSnapshot extends AsyncCollectionSpecificAdminRequest {
protected final String commitName;
public CreateSnapshot(String collection, String commitName) {
super(CollectionAction.CREATESNAPSHOT, checkNotNull(CoreAdminParams.COLLECTION ,collection));
this.commitName = checkNotNull(CoreAdminParams.COMMIT_NAME, commitName);
}
public String getCollectionName() {
return collection;
}
public String getCommitName() {
return commitName;
}
@Override
public AsyncCollectionSpecificAdminRequest setCollectionName (String collection) {
this.collection = checkNotNull(CoreAdminParams.COLLECTION ,collection);
return this;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CoreAdminParams.COLLECTION, collection);
params.set(CoreAdminParams.COMMIT_NAME, commitName);
return params;
}
}
@SuppressWarnings("serial")
public static class DeleteSnapshot extends AsyncCollectionSpecificAdminRequest {
protected final String commitName;
public DeleteSnapshot (String collection, String commitName) {
super(CollectionAction.DELETESNAPSHOT, checkNotNull(CoreAdminParams.COLLECTION ,collection));
this.commitName = checkNotNull(CoreAdminParams.COMMIT_NAME, commitName);
}
public String getCollectionName() {
return collection;
}
public String getCommitName() {
return commitName;
}
@Override
public AsyncCollectionSpecificAdminRequest setCollectionName (String collection) {
this.collection = checkNotNull(CoreAdminParams.COLLECTION ,collection);
return this;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CoreAdminParams.COLLECTION, collection);
params.set(CoreAdminParams.COMMIT_NAME, commitName);
return params;
}
}
@SuppressWarnings("serial")
public static class ListSnapshots extends AsyncCollectionSpecificAdminRequest {
public ListSnapshots (String collection) {
super(CollectionAction.LISTSNAPSHOTS, checkNotNull(CoreAdminParams.COLLECTION ,collection));
}
public String getCollectionName() {
return collection;
}
@Override
public AsyncCollectionSpecificAdminRequest setCollectionName (String collection) {
this.collection = checkNotNull(CoreAdminParams.COLLECTION ,collection);
return this;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set(CoreAdminParams.COLLECTION, collection);
return params;
}
}
/**
* Returns a SolrRequest to create a new shard in a collection
*/

3
solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
View File

@ -91,6 +91,9 @@ public interface CollectionParams {
MIGRATESTATEFORMAT(true, LockLevel.CLUSTER),
BACKUP(true, LockLevel.COLLECTION),
RESTORE(true, LockLevel.COLLECTION),
CREATESNAPSHOT(true, LockLevel.COLLECTION),
DELETESNAPSHOT(true, LockLevel.COLLECTION),
LISTSNAPSHOTS(false, LockLevel.NONE),
//only for testing. it just waits for specified time
// these are not exposed via collection API commands
// but the overseer is aware of these tasks