mirror of https://github.com/apache/lucene.git
SOLR-9038: add snapshotscli.sh and supporting code
This commit is contained in:
yonik
parent
55ed23168f
commit
46f5f45862
|
|
@ -50,6 +50,8 @@ New Features
|
|||
* SOLR-9682: JSON Facet API: added "param" query type to facet domain filter specification to obtain
|
||||
filters via query parameters. (yonik)
|
||||
|
||||
* SOLR-9038: Add a command-line tool to manage the snapshots functionality (Hrishikesh Gadre via yonik)
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
* SOLR-9704: Facet Module / JSON Facet API: Optimize blockChildren facets that have
|
||||
|
|
|
|||
|
|
@ -0,0 +1,468 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.core.snapshots;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.CommandLineParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.commons.cli.PosixParser;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.params.CollectionAdminParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
/**
|
||||
* This class provides utility functions required for Solr snapshots functionality.
|
||||
*/
|
||||
public class SolrSnapshotsTool implements Closeable {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
|
||||
|
||||
private static final String CREATE = "create";
|
||||
private static final String DELETE = "delete";
|
||||
private static final String LIST = "list";
|
||||
private static final String DESCRIBE = "describe";
|
||||
private static final String PREPARE_FOR_EXPORT = "prepare-snapshot-export";
|
||||
private static final String EXPORT_SNAPSHOT = "export";
|
||||
private static final String HELP = "help";
|
||||
private static final String COLLECTION = "c";
|
||||
private static final String TEMP_DIR = "t";
|
||||
private static final String DEST_DIR = "d";
|
||||
private static final String SOLR_ZK_ENSEMBLE = "z";
|
||||
private static final String HDFS_PATH_PREFIX = "p";
|
||||
private static final String BACKUP_REPO_NAME = "r";
|
||||
private static final String ASYNC_REQ_ID = "i";
|
||||
private static final List<String> OPTION_HELP_ORDER = Arrays.asList(CREATE, DELETE, LIST, DESCRIBE,
|
||||
PREPARE_FOR_EXPORT, EXPORT_SNAPSHOT, HELP, SOLR_ZK_ENSEMBLE, COLLECTION, DEST_DIR, BACKUP_REPO_NAME,
|
||||
ASYNC_REQ_ID, TEMP_DIR, HDFS_PATH_PREFIX);
|
||||
|
||||
private final CloudSolrClient solrClient;
|
||||
|
||||
public SolrSnapshotsTool(String solrZkEnsemble) {
|
||||
solrClient = (new CloudSolrClient.Builder()).withZkHost(solrZkEnsemble).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (solrClient != null) {
|
||||
solrClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void createSnapshot(String collectionName, String snapshotName) {
|
||||
CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);
|
||||
CollectionAdminResponse resp;
|
||||
try {
|
||||
resp = createSnap.process(solrClient);
|
||||
Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
|
||||
System.out.println("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
||||
System.out.println("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName
|
||||
+" due to following error : "+e.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteSnapshot(String collectionName, String snapshotName) {
|
||||
CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);
|
||||
CollectionAdminResponse resp;
|
||||
try {
|
||||
resp = deleteSnap.process(solrClient);
|
||||
Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
|
||||
System.out.println("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
||||
System.out.println("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName
|
||||
+" due to following error : "+e.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
public void listSnapshots(String collectionName) {
|
||||
CollectionAdminRequest.ListSnapshots listSnaps = new CollectionAdminRequest.ListSnapshots(collectionName);
|
||||
CollectionAdminResponse resp;
|
||||
try {
|
||||
resp = listSnaps.process(solrClient);
|
||||
Preconditions.checkState(resp.getStatus() == 0, "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());
|
||||
|
||||
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
|
||||
for (int i = 0; i < apiResult.size(); i++) {
|
||||
System.out.println(apiResult.getName(i));
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to list snapshots for collection " + collectionName, e);
|
||||
System.out.println("Failed to list snapshots for collection " + collectionName
|
||||
+" due to following error : "+e.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void describeSnapshot(String collectionName, String snapshotName) {
|
||||
try {
|
||||
Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
|
||||
for (CollectionSnapshotMetaData m : snaps) {
|
||||
if (snapshotName.equals(m.getName())) {
|
||||
System.out.println("Name: " + m.getName());
|
||||
System.out.println("Status: " + m.getStatus());
|
||||
System.out.println("Time of creation: " + dateFormat.format(m.getCreationDate()));
|
||||
System.out.println("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());
|
||||
System.out.println("-----------------------------------");
|
||||
for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("Core [name=");
|
||||
builder.append(n.getCoreName());
|
||||
builder.append(", leader=");
|
||||
builder.append(n.isLeader());
|
||||
builder.append(", generation=");
|
||||
builder.append(n.getGenerationNumber());
|
||||
builder.append(", indexDirPath=");
|
||||
builder.append(n.getIndexDirPath());
|
||||
builder.append("]\n");
|
||||
System.out.println(builder.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to fetch snapshot details", e);
|
||||
System.out.println("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getIndexFilesPathForSnapshot(String collectionName, String snapshotName, Optional<String> pathPrefix)
|
||||
throws SolrServerException, IOException {
|
||||
Map<String, List<String>> result = new HashMap<>();
|
||||
|
||||
Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
|
||||
Optional<CollectionSnapshotMetaData> meta = Optional.empty();
|
||||
for (CollectionSnapshotMetaData m : snaps) {
|
||||
if (snapshotName.equals(m.getName())) {
|
||||
meta = Optional.of(m);
|
||||
}
|
||||
}
|
||||
|
||||
if (!meta.isPresent()) {
|
||||
throw new IllegalArgumentException("The snapshot named " + snapshotName
|
||||
+ " is not found for collection " + collectionName);
|
||||
}
|
||||
|
||||
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
|
||||
for (Slice s : collectionState.getSlices()) {
|
||||
List<CoreSnapshotMetaData> replicaSnaps = meta.get().getReplicaSnapshotsForShard(s.getName());
|
||||
// Prepare a list of *existing* replicas (since one or more replicas could have been deleted after the snapshot creation).
|
||||
List<CoreSnapshotMetaData> availableReplicas = new ArrayList<>();
|
||||
for (CoreSnapshotMetaData m : replicaSnaps) {
|
||||
if (isReplicaAvailable(s, m.getCoreName())) {
|
||||
availableReplicas.add(m);
|
||||
}
|
||||
}
|
||||
|
||||
if (availableReplicas.isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"The snapshot named " + snapshotName + " not found for shard "
|
||||
+ s.getName() + " of collection " + collectionName);
|
||||
}
|
||||
|
||||
// Prefer a leader replica (at the time when the snapshot was created).
|
||||
CoreSnapshotMetaData coreSnap = availableReplicas.get(0);
|
||||
for (CoreSnapshotMetaData m : availableReplicas) {
|
||||
if (m.isLeader()) {
|
||||
coreSnap = m;
|
||||
}
|
||||
}
|
||||
|
||||
String indexDirPath = coreSnap.getIndexDirPath();
|
||||
if (pathPrefix.isPresent()) {
|
||||
// If the path prefix is specified, rebuild the path to the index directory.
|
||||
Path t = new Path(coreSnap.getIndexDirPath());
|
||||
indexDirPath = (new Path(pathPrefix.get(), t.toUri().getPath())).toString();
|
||||
}
|
||||
|
||||
List<String> paths = new ArrayList<>();
|
||||
for (String fileName : coreSnap.getFiles()) {
|
||||
Path p = new Path(indexDirPath, fileName);
|
||||
paths.add(p.toString());
|
||||
}
|
||||
|
||||
result.put(s.getName(), paths);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public void buildCopyListings(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix)
|
||||
throws SolrServerException, IOException {
|
||||
Map<String, List<String>> paths = getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix);
|
||||
for (Map.Entry<String,List<String>> entry : paths.entrySet()) {
|
||||
StringBuilder filesBuilder = new StringBuilder();
|
||||
for (String filePath : entry.getValue()) {
|
||||
filesBuilder.append(filePath);
|
||||
filesBuilder.append("\n");
|
||||
}
|
||||
|
||||
String files = filesBuilder.toString().trim();
|
||||
try (Writer w = new OutputStreamWriter(new FileOutputStream(new File(localFsPath, entry.getKey())), StandardCharsets.UTF_8)) {
|
||||
w.write(files);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc) throws SolrServerException, IOException {
|
||||
// Backup the collection meta-data
|
||||
CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
|
||||
backup.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
|
||||
backup.setLocation(backupLoc);
|
||||
CollectionAdminResponse resp = backup.process(solrClient);
|
||||
Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
|
||||
}
|
||||
|
||||
public void prepareForExport(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix, String destPath) {
|
||||
try {
|
||||
buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
|
||||
System.out.println("Successfully prepared copylisting for the snapshot export.");
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
||||
+ collectionName, e);
|
||||
System.out.println("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
||||
+ collectionName + " due to following error : " + e.getLocalizedMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
backupCollectionMetaData(collectionName, snapshotName, destPath);
|
||||
System.out.println("Successfully backed up collection meta-data");
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
||||
System.out.println("Failed to backup collection meta-data for collection " + collectionName
|
||||
+ " due to following error : " + e.getLocalizedMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
public void exportSnapshot(String collectionName, String snapshotName, String destPath, Optional<String> backupRepo,
|
||||
Optional<String> asyncReqId) {
|
||||
try {
|
||||
CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
|
||||
backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);
|
||||
backup.setLocation(destPath);
|
||||
if (backupRepo.isPresent()) {
|
||||
backup.setRepositoryName(backupRepo.get());
|
||||
}
|
||||
if (asyncReqId.isPresent()) {
|
||||
backup.setAsyncId(asyncReqId.get());
|
||||
}
|
||||
CollectionAdminResponse resp = backup.process(solrClient);
|
||||
Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
||||
System.out.println("Failed to backup collection meta-data for collection " + collectionName
|
||||
+ " due to following error : " + e.getLocalizedMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
CommandLineParser parser = new PosixParser();
|
||||
Options options = new Options();
|
||||
|
||||
options.addOption(null, CREATE, true, "This command will create a snapshot with the specified name");
|
||||
options.addOption(null, DELETE, true, "This command will delete a snapshot with the specified name");
|
||||
options.addOption(null, LIST, false, "This command will list all the named snapshots for the specified collection.");
|
||||
options.addOption(null, DESCRIBE, true, "This command will print details for a named snapshot for the specified collection.");
|
||||
options.addOption(null, PREPARE_FOR_EXPORT, true, "This command will prepare copylistings for the specified snapshot."
|
||||
+ " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared"
|
||||
+ " file-system e.g. HDFS");
|
||||
options.addOption(null, EXPORT_SNAPSHOT, true, "This command will create a backup for the specified snapshot.");
|
||||
options.addOption(null, HELP, false, "This command will print the help message for the snapshots related commands.");
|
||||
options.addOption(TEMP_DIR, true, "This parameter specifies the path of a temporary directory on local filesystem"
|
||||
+ " during prepare-snapshot-export command.");
|
||||
options.addOption(DEST_DIR, true, "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related"
|
||||
+ " information should be stored.");
|
||||
options.addOption(COLLECTION, true, "This parameter specifies the name of the collection to be used during snapshot operation");
|
||||
options.addOption(SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address");
|
||||
options.addOption(HDFS_PATH_PREFIX, true, "This parameter specifies the HDFS URI prefix to be used"
|
||||
+ " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.");
|
||||
options.addOption(BACKUP_REPO_NAME, true, "This parameter specifies the name of the backup repository to be used"
|
||||
+ " during snapshot export preparation");
|
||||
options.addOption(ASYNC_REQ_ID, true, "This parameter specifies the async request identifier to be used"
|
||||
+ " during snapshot export preparation");
|
||||
|
||||
CommandLine cmd = null;
|
||||
try {
|
||||
cmd = parser.parse(options, args);
|
||||
} catch (ParseException e) {
|
||||
System.out.println(e.getLocalizedMessage());
|
||||
printHelp(options);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
if (cmd.hasOption(CREATE) || cmd.hasOption(DELETE) || cmd.hasOption(LIST) || cmd.hasOption(DESCRIBE)
|
||||
|| cmd.hasOption(PREPARE_FOR_EXPORT) || cmd.hasOption(EXPORT_SNAPSHOT)) {
|
||||
try (SolrSnapshotsTool tool = new SolrSnapshotsTool(cmd.getOptionValue(SOLR_ZK_ENSEMBLE))) {
|
||||
if (cmd.hasOption(CREATE)) {
|
||||
String snapshotName = cmd.getOptionValue(CREATE);
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
tool.createSnapshot(collectionName, snapshotName);
|
||||
|
||||
} else if (cmd.hasOption(DELETE)) {
|
||||
String snapshotName = cmd.getOptionValue(DELETE);
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
tool.deleteSnapshot(collectionName, snapshotName);
|
||||
|
||||
} else if (cmd.hasOption(LIST)) {
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
tool.listSnapshots(collectionName);
|
||||
|
||||
} else if (cmd.hasOption(DESCRIBE)) {
|
||||
String snapshotName = cmd.getOptionValue(DESCRIBE);
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
tool.describeSnapshot(collectionName, snapshotName);
|
||||
|
||||
} else if (cmd.hasOption(PREPARE_FOR_EXPORT)) {
|
||||
String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT);
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
String localFsDir = requiredArg(options, cmd, TEMP_DIR);
|
||||
String hdfsOpDir = requiredArg(options, cmd, DEST_DIR);
|
||||
Optional<String> pathPrefix = Optional.ofNullable(cmd.getOptionValue(HDFS_PATH_PREFIX));
|
||||
|
||||
if (pathPrefix.isPresent()) {
|
||||
try {
|
||||
new URI(pathPrefix.get());
|
||||
} catch (URISyntaxException e) {
|
||||
System.out.println(
|
||||
"The specified File system path prefix " + pathPrefix.get()
|
||||
+ " is invalid. The error is " + e.getLocalizedMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir);
|
||||
|
||||
} else if (cmd.hasOption(EXPORT_SNAPSHOT)) {
|
||||
String snapshotName = cmd.getOptionValue(EXPORT_SNAPSHOT);
|
||||
String collectionName = cmd.getOptionValue(COLLECTION);
|
||||
String destDir = requiredArg(options, cmd, DEST_DIR);
|
||||
Optional<String> backupRepo = Optional.ofNullable(cmd.getOptionValue(BACKUP_REPO_NAME));
|
||||
Optional<String> asyncReqId = Optional.ofNullable(cmd.getOptionValue(ASYNC_REQ_ID));
|
||||
|
||||
tool.exportSnapshot(collectionName, snapshotName, destDir, backupRepo, asyncReqId);
|
||||
}
|
||||
}
|
||||
} else if (cmd.hasOption(HELP)) {
|
||||
printHelp(options);
|
||||
} else {
|
||||
System.out.println("Unknown command specified.");
|
||||
printHelp(options);
|
||||
}
|
||||
}
|
||||
|
||||
private static String requiredArg(Options options, CommandLine cmd, String optVal) {
|
||||
if (!cmd.hasOption(optVal)) {
|
||||
System.out.println("Please specify the value for option " + optVal);
|
||||
printHelp(options);
|
||||
System.exit(1);
|
||||
}
|
||||
return cmd.getOptionValue(optVal);
|
||||
}
|
||||
|
||||
private static boolean isReplicaAvailable (Slice s, String coreName) {
|
||||
for (Replica r: s.getReplicas()) {
|
||||
if (coreName.equals(r.getCoreName())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(String collectionName)
|
||||
throws SolrServerException, IOException {
|
||||
CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots(collectionName);
|
||||
CollectionAdminResponse resp = listSnapshots.process(solrClient);
|
||||
|
||||
Preconditions.checkState(resp.getStatus() == 0);
|
||||
|
||||
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
|
||||
|
||||
Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
|
||||
for (int i = 0; i < apiResult.size(); i++) {
|
||||
result.add(new CollectionSnapshotMetaData((NamedList<Object>)apiResult.getVal(i)));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void printHelp(Options options) {
|
||||
StringBuilder helpFooter = new StringBuilder();
|
||||
helpFooter.append("Examples: \n");
|
||||
helpFooter.append("snapshotscli.sh --create snapshot-1 -c books -z localhost:2181 \n");
|
||||
helpFooter.append("snapshotscli.sh --list -c books -z localhost:2181 \n");
|
||||
helpFooter.append("snapshotscli.sh --describe snapshot-1 -c books -z localhost:2181 \n");
|
||||
helpFooter.append("snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n");
|
||||
helpFooter.append("snapshotscli.sh --delete snapshot-1 -c books -z localhost:2181 \n");
|
||||
|
||||
HelpFormatter formatter = new HelpFormatter();
|
||||
formatter.setOptionComparator(new OptionComarator<>());
|
||||
formatter.printHelp("SolrSnapshotsTool", null, options, helpFooter.toString(), false);
|
||||
}
|
||||
|
||||
private static class OptionComarator<T extends Option> implements Comparator<T> {
|
||||
|
||||
public int compare(T o1, T o2) {
|
||||
String s1 = o1.hasLongOpt() ? o1.getLongOpt() : o1.getOpt();
|
||||
String s2 = o2.hasLongOpt() ? o2.getLongOpt() : o2.getOpt();
|
||||
return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
run_solr_snapshot_tool() {
|
||||
JVM="java"
|
||||
scriptDir=$(dirname "$0")
|
||||
if [ -n "$LOG4J_PROPS" ]; then
|
||||
log4j_config="file:${LOG4J_PROPS}"
|
||||
else
|
||||
log4j_config="file:${scriptDir}/log4j.properties"
|
||||
fi
|
||||
PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configuration=${log4j_config} \
|
||||
-classpath "${solrLibPath}" org.apache.solr.core.snapshots.SolrSnapshotsTool "$@" 2> /dev/null
|
||||
}
|
||||
|
||||
usage() {
|
||||
run_solr_snapshot_tool --help
|
||||
}
|
||||
|
||||
distcp_warning() {
|
||||
echo "SOLR_USE_DISTCP environment variable is not set. \
|
||||
Do you want to use hadoop distcp tool for exporting Solr collection snapshot ?"
|
||||
}
|
||||
|
||||
parse_options() {
|
||||
OPTIND=3
|
||||
while getopts ":c:d:s:z:p:r:i:" o ; do
|
||||
case "${o}" in
|
||||
d)
|
||||
destPath=${OPTARG}
|
||||
;;
|
||||
s)
|
||||
sourcePath=${OPTARG}
|
||||
;;
|
||||
c)
|
||||
collectionName=${OPTARG}
|
||||
;;
|
||||
z)
|
||||
solrZkEnsemble=${OPTARG}
|
||||
;;
|
||||
p)
|
||||
pathPrefix=${OPTARG}
|
||||
;;
|
||||
r)
|
||||
backupRepoName=${OPTARG}
|
||||
;;
|
||||
i)
|
||||
aysncReqId=${OPTARG}
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option ${OPTARG}"
|
||||
usage 1>&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
prepare_snapshot_export() {
|
||||
#Make sure to cleanup the temporary files.
|
||||
scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)
|
||||
function finish {
|
||||
rm -rf "${scratch}"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
if hdfs dfs -test -d "${destPath}" ; then
|
||||
run_solr_snapshot_tool --prepare-snapshot-export "$@" -t "${scratch}"
|
||||
|
||||
hdfs dfs -mkdir -p "${copyListingDirPath}" > /dev/null
|
||||
find "${scratch}" -type f -printf "%f\n" | while read shardId; do
|
||||
echo "Copying the copy-listing for $shardId"
|
||||
hdfs dfs -copyFromLocal "${scratch}/${shardId}" "${copyListingDirPath}" > /dev/null
|
||||
done
|
||||
else
|
||||
echo "Directory ${destPath} does not exist."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
copy_snapshot_files() {
|
||||
copylisting_dir_path="$1"
|
||||
|
||||
if hdfs dfs -test -d "${copylisting_dir_path}" ; then
|
||||
for shardId in $(hdfs dfs -stat "%n" "${copylisting_dir_path}/*"); do
|
||||
oPath="${destPath}/${snapshotName}/snapshot.${shardId}"
|
||||
echo "Copying the index files for ${shardId} to ${oPath}"
|
||||
${distCpCmd} -f " ${copylisting_dir_path}/${shardId}" "${oPath}" > /dev/null
|
||||
done
|
||||
else
|
||||
echo "Directory ${copylisting_dir_path} does not exist."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
collectionName=""
|
||||
solrZkEnsemble=""
|
||||
pathPrefix=""
|
||||
destPath=""
|
||||
sourcePath=""
|
||||
cmd="$1"
|
||||
snapshotName="$2"
|
||||
copyListingDirPath=""
|
||||
distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"
|
||||
scriptDir=$(dirname "$0")
|
||||
solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../lib/ext/*}"
|
||||
|
||||
case "${cmd}" in
|
||||
--create)
|
||||
run_solr_snapshot_tool "$@"
|
||||
;;
|
||||
--delete)
|
||||
run_solr_snapshot_tool "$@"
|
||||
;;
|
||||
--list)
|
||||
run_solr_snapshot_tool "$@"
|
||||
;;
|
||||
--describe)
|
||||
run_solr_snapshot_tool "$@"
|
||||
;;
|
||||
--prepare-snapshot-export)
|
||||
: "${SOLR_USE_DISTCP:? $(distcp_warning)}"
|
||||
|
||||
parse_options "$@"
|
||||
|
||||
: "${destPath:? Please specify destination directory using -d option}"
|
||||
|
||||
copyListingDirPath="${destPath}/copylistings"
|
||||
prepare_snapshot_export "${@:2}"
|
||||
echo "Done. GoodBye!"
|
||||
;;
|
||||
--export)
|
||||
if [ -z "${SOLR_USE_DISTCP}" ]; then
|
||||
run_solr_snapshot_tool "$@"
|
||||
echo "Done. GoodBye!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
parse_options "$@"
|
||||
|
||||
: "${snapshotName:? Please specify the name of the snapshot}"
|
||||
: "${destPath:? Please specify destination directory using -d option}"
|
||||
|
||||
if [ -n "${collectionName}" ] && [ -n "${sourcePath}" ]; then
|
||||
echo "The -c and -s options can not be specified together"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${collectionName}" ] && [ -z "${sourcePath}" ]; then
|
||||
echo "At least one of options (-c or -s) must be specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${collectionName}" ]; then
|
||||
copyListingDirPath="${destPath}/${snapshotName}/copylistings"
|
||||
prepare_snapshot_export "${@:2}"
|
||||
copy_snapshot_files "${destPath}/${snapshotName}/copylistings"
|
||||
hdfs dfs -rm -r -f -skipTrash "${destPath}/${snapshotName}/copylistings" > /dev/null
|
||||
else
|
||||
copy_snapshot_files "${sourcePath}/copylistings"
|
||||
echo "Copying the collection meta-data to ${destPath}/${snapshotName}"
|
||||
${distCpCmd} "${sourcePath}/${snapshotName}/*" "${destPath}/${snapshotName}/" > /dev/null
|
||||
fi
|
||||
|
||||
echo "Done. GoodBye!"
|
||||
;;
|
||||
--help)
|
||||
usage 1>&2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown command ${cmd}"
|
||||
usage 1>&2
|
||||
exit 1
|
||||
esac
|
||||
|
||||
Loading…
Reference in New Issue