From 1381dd9287a23c950eaaa3c258249a5ebc812f35 Mon Sep 17 00:00:00 2001 From: markrmiller Date: Fri, 4 Nov 2016 10:41:52 -0400 Subject: [PATCH] SOLR-9055: Make collection backup/restore extensible. - Introduced a parameter for the Backup operation to specify index backup strategy. - Introduced two strategies for backing up index data. - One using core Admin API (BACKUPCORE) - Other skipping the backup of index data altogether. This is useful when the index data is copied via an external mechanism in combination with named snapshots (Please refer to SOLR-9038 for details) - In future we can add additional implementations of this interface (e.g. based on HDFS snapshots etc.) - Added a backup property to record the Solr version. This helps to check the compatibility of backup with respect to the current version during the restore operation. This compatibility check is not added since its unclear what the Solr level compatibility guidelines are. But at-least having version information as part of the backup would be very useful. --- solr/CHANGES.txt | 2 + .../java/org/apache/solr/cloud/BackupCmd.java | 140 ++++++++++-------- .../org/apache/solr/cloud/RestoreCmd.java | 2 +- .../solr/core/backup/BackupManager.java | 3 +- .../handler/admin/CollectionsHandler.java | 6 + .../AbstractCloudBackupRestoreTestCase.java | 16 ++ .../cloud/TestHdfsCloudBackupRestore.java | 56 +++++++ .../solrj/request/CollectionAdminRequest.java | 14 ++ .../common/params/CollectionAdminParams.java | 24 +++ 9 files changed, 201 insertions(+), 62 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index cfe045da50b..7c0d211c1a6 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -96,6 +96,8 @@ New Features * SOLR-8542: Adds Solr Learning to Rank (LTR) plugin for reranking results with machine learning models. (Michael Nilsson, Diego Ceccarelli, Joshua Pantony, Jon Dorando, Naveen Santhapuri, Alessandro Benedetti, David Grohmann, Christine Poerschke) +* SOLR-9055: Make collection backup/restore extensible. (Hrishikesh Gadre, Varun Thacker, Mark Miller) + Optimizations ---------------------- * SOLR-9704: Facet Module / JSON Facet API: Optimize blockChildren facets that have diff --git a/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java index b859d8ea0d6..a4012f05fc7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/BackupCmd.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; +import org.apache.lucene.util.Version; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ClusterState; @@ -35,6 +36,7 @@ import org.apache.solr.common.cloud.Replica.State; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -68,31 +70,13 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd { public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { String collectionName = message.getStr(COLLECTION_PROP); String backupName = message.getStr(NAME); - ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); - String asyncId = message.getStr(ASYNC); String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY); - String commitName = message.getStr(CoreAdminParams.COMMIT_NAME); - Optional snapshotMeta = Optional.empty(); - if (commitName != null) { - SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient(); - snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName); - if (!snapshotMeta.isPresent()) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName - + " does not exist for collection " + collectionName); - } - if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName - + " has not completed successfully. The status is " + snapshotMeta.get().getStatus()); - } - } - - Map requestMap = new HashMap<>(); Instant startTime = Instant.now(); CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer(); BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo)); - BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader, collectionName); + BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader); // Backup location URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); @@ -106,50 +90,16 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd { // Create a directory to store backup details. repository.createDirectory(backupPath); - log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, - backupPath); - - Collection shardsToConsider = Collections.emptySet(); - if (snapshotMeta.isPresent()) { - shardsToConsider = snapshotMeta.get().getShards(); - } - - for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) { - Replica replica = null; - - if (snapshotMeta.isPresent()) { - if (!shardsToConsider.contains(slice.getName())) { - log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.", - slice.getName(), collectionName, snapshotMeta.get().getName()); - continue; - } - replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice); - } else { - // Note - Actually this can return a null value when there is no leader for this shard. - replica = slice.getLeader(); - if (replica == null) { - throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName); - } + String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY); + switch (strategy) { + case CollectionAdminParams.COPY_FILES_STRATEGY: { + copyIndexFiles(backupPath, message, results); + break; } - - String coreName = replica.getStr(CORE_NAME_PROP); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString()); - params.set(NAME, slice.getName()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name - params.set(CORE_NAME_PROP, coreName); - if (snapshotMeta.isPresent()) { - params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName()); + case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: { + break; } - - ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap); - log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); } - log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); - - ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap); log.info("Starting to backup ZK data for backupName={}", backupName); @@ -168,6 +118,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd { properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName); properties.put(COLL_CONF, configName); properties.put(BackupManager.START_TIME_PROP, startTime.toString()); + properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString()); //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same. //if they are not the same then we can throw an error or have an 'overwriteConfig' flag //TODO save numDocs for the shardLeader. We can use it to sanity check the restore. @@ -202,4 +153,73 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd { return r.get(); } + + private void copyIndexFiles(URI backupPath, ZkNodeProps request, NamedList results) throws Exception { + String collectionName = request.getStr(COLLECTION_PROP); + String backupName = request.getStr(NAME); + String asyncId = request.getStr(ASYNC); + String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY); + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); + Map requestMap = new HashMap<>(); + + String commitName = request.getStr(CoreAdminParams.COMMIT_NAME); + Optional snapshotMeta = Optional.empty(); + if (commitName != null) { + SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient(); + snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName); + if (!snapshotMeta.isPresent()) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + + " does not exist for collection " + collectionName); + } + if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName + + " has not completed successfully. The status is " + snapshotMeta.get().getStatus()); + } + } + + log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, + backupPath); + + Collection shardsToConsider = Collections.emptySet(); + if (snapshotMeta.isPresent()) { + shardsToConsider = snapshotMeta.get().getShards(); + } + + for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) { + Replica replica = null; + + if (snapshotMeta.isPresent()) { + if (!shardsToConsider.contains(slice.getName())) { + log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.", + slice.getName(), collectionName, snapshotMeta.get().getName()); + continue; + } + replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice); + } else { + // Note - Actually this can return a null value when there is no leader for this shard. + replica = slice.getLeader(); + if (replica == null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName); + } + } + + String coreName = replica.getStr(CORE_NAME_PROP); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString()); + params.set(NAME, slice.getName()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName); + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name + params.set(CORE_NAME_PROP, coreName); + if (snapshotMeta.isPresent()) { + params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName()); + } + + ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap); + log.debug("Sent backup request to core={} for backupName={}", coreName, backupName); + } + log.debug("Sent backup requests to all shard leaders for backupName={}", backupName); + + ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap); + } } diff --git a/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java index 63d56865700..4e7fb581b34 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java @@ -87,7 +87,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION)); URI backupPath = repository.resolve(location, backupName); ZkStateReader zkStateReader = ocmh.zkStateReader; - BackupManager backupMgr = new BackupManager(repository, zkStateReader, restoreCollectionName); + BackupManager backupMgr = new BackupManager(repository, zkStateReader); Properties properties = backupMgr.readBackupProperties(location, backupName); String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP); diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java index c80b2b7c279..726e5b9799c 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java @@ -68,7 +68,7 @@ public class BackupManager { protected final ZkStateReader zkStateReader; protected final BackupRepository repository; - public BackupManager(BackupRepository repository, ZkStateReader zkStateReader, String collectionName) { + public BackupManager(BackupRepository repository, ZkStateReader zkStateReader) { this.repository = Objects.requireNonNull(repository); this.zkStateReader = Objects.requireNonNull(zkStateReader); } @@ -126,6 +126,7 @@ public class BackupManager { * * @param backupLoc The base path used to store the backup data. * @param backupId The unique name for the backup. + * @param collectionName The name of the collection whose meta-data is to be returned. * @return the meta-data information for the backed-up collection. * @throws IOException in case of errors. */ diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index e290ccb5ca2..01095a1143b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -734,8 +734,14 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex); } + String strategy = req.getParams().get(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY); + if (!CollectionAdminParams.INDEX_BACKUP_STRATEGIES.contains(strategy)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown index backup strategy " + strategy); + } + Map params = req.getParams().getAll(null, NAME, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME); params.put(CoreAdminParams.BACKUP_LOCATION, location); + params.put(CollectionAdminParams.INDEX_BACKUP_STRATEGY, strategy); return params; }), RESTORE_OP(RESTORE, (req, rsp, h) -> { diff --git a/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java b/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java index c7b27450938..f39cfed48d8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java +++ b/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java @@ -38,6 +38,7 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.ImplicitDocRouter; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; import org.junit.BeforeClass; import org.junit.Test; @@ -124,9 +125,24 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa } testBackupAndRestore(getCollectionName()); + testConfigBackupOnly("conf1", getCollectionName()); testInvalidPath(getCollectionName()); } + /** + * This test validates the backup of collection configuration using + * {@linkplain CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}. + * + * @param configName The config name for the collection to be backed up. + * @param collectionName The name of the collection to be backed up. + * @throws Exception in case of errors. + */ + protected void testConfigBackupOnly(String configName, String collectionName) throws Exception { + // This is deliberately no-op since we want to run this test only for one of the backup repository + // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository test + // implements this. + } + // This test verifies the system behavior when the backup location cluster property is configured with an invalid // value for the specified repository (and the default backup location is not configured in solr.xml). private void testInvalidPath(String collectionName) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/TestHdfsCloudBackupRestore.java index 5fd7666d411..40a6e30f9d8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestHdfsCloudBackupRestore.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestHdfsCloudBackupRestore.java @@ -16,10 +16,18 @@ */ package org.apache.solr.cloud; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.core.backup.BackupManager.*; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.commons.io.IOUtils; @@ -28,7 +36,14 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.hdfs.HdfsTestUtil; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.backup.BackupManager; +import org.apache.solr.core.backup.repository.HdfsBackupRepository; import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -144,4 +159,45 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa public String getBackupLocation() { return null; } + + protected void testConfigBackupOnly(String configName, String collectionName) throws Exception { + String backupName = "configonlybackup"; + CloudSolrClient solrClient = cluster.getSolrClient(); + + CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName) + .setRepositoryName(getBackupRepoName()) + .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY); + backup.process(solrClient); + + Map params = new HashMap<>(); + params.put("location", "/backup"); + params.put("solr.hdfs.home", hdfsUri + "/solr"); + + HdfsBackupRepository repo = new HdfsBackupRepository(); + repo.init(new NamedList<>(params)); + BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader()); + + URI baseLoc = repo.createURI("/backup"); + + Properties props = mgr.readBackupProperties(baseLoc, backupName); + assertNotNull(props); + assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP)); + assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP)); + assertEquals(configName, props.getProperty(COLL_CONF)); + + DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName); + assertNotNull(collectionState); + assertEquals(collectionName, collectionState.getName()); + + URI configDirLoc = repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName); + assertTrue(repo.exists(configDirLoc)); + + Collection expected = Arrays.asList(BACKUP_PROPS_FILE, ZK_STATE_DIR); + URI backupLoc = repo.resolve(baseLoc, backupName); + String[] dirs = repo.listAll(backupLoc); + for (String d : dirs) { + assertTrue(expected.contains(d)); + } + } + } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 92ea99bc0b8..0beaa55b644 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -713,10 +713,12 @@ public abstract class CollectionAdminRequest protected Optional repositoryName = Optional.empty(); protected String location; protected Optional commitName = Optional.empty(); + protected Optional indexBackupStrategy = Optional.empty(); public Backup(String collection, String name) { super(CollectionAction.BACKUP, collection); this.name = name; + this.repositoryName = Optional.empty(); } @Override @@ -760,6 +762,15 @@ public abstract class CollectionAdminRequest return this; } + public Optional getIndexBackupStrategy() { + return indexBackupStrategy; + } + + public Backup setIndexBackupStrategy(String indexBackupStrategy) { + this.indexBackupStrategy = Optional.ofNullable(indexBackupStrategy); + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); @@ -772,6 +783,9 @@ public abstract class CollectionAdminRequest if (commitName.isPresent()) { params.set(CoreAdminParams.COMMIT_NAME, commitName.get()); } + if (indexBackupStrategy.isPresent()) { + params.set(CollectionAdminParams.INDEX_BACKUP_STRATEGY, indexBackupStrategy.get()); + } return params; } diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index a8686a12322..98ae2928b09 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -16,6 +16,9 @@ */ package org.apache.solr.common.params; +import java.util.Arrays; +import java.util.Collection; + public interface CollectionAdminParams { /* Param used by DELETESTATUS call to clear all stored responses */ @@ -26,4 +29,25 @@ public interface CollectionAdminParams { String COUNT_PROP = "count"; + /** + * A parameter to specify the name of the index backup strategy to be used. + */ + public static final String INDEX_BACKUP_STRATEGY = "indexBackup"; + + /** + * This constant defines the index backup strategy based on copying index files to desired location. + */ + public static final String COPY_FILES_STRATEGY = "copy-files"; + + /** + * This constant defines the strategy to not copy index files (useful for meta-data only backup). + */ + public static final String NO_INDEX_BACKUP_STRATEGY = "none"; + + /** + * This constant defines a list of valid index backup strategies. + */ + public static final Collection INDEX_BACKUP_STRATEGIES = + Arrays.asList(COPY_FILES_STRATEGY, NO_INDEX_BACKUP_STRATEGY); + }