SOLR-9055: Make collection backup/restore extensible.

- Introduced a parameter for the Backup operation to specify index backup strategy.
- Introduced two strategies for backing up index data.
  - One using core Admin API (BACKUPCORE)
  - Other skipping the backup of index data altogether. This is useful when
    the index data is copied via an external mechanism in combination with named
    snapshots (Please refer to SOLR-9038 for details)
  - In future we can add additional implementations of this interface (e.g. based on HDFS snapshots etc.)
- Added a backup property to record the Solr version. This helps to check the compatibility
  of backup with respect to the current version during the restore operation. This
  compatibility check is not added since its unclear what the Solr level compatibility guidelines
  are. But at-least having version information as part of the backup would be very useful.

solr/CHANGES.txt

@@ -96,6 +96,8 @@ New Features
 * SOLR-8542: Adds Solr Learning to Rank (LTR) plugin for reranking results with machine learning models.
   (Michael Nilsson, Diego Ceccarelli, Joshua Pantony, Jon Dorando, Naveen Santhapuri, Alessandro Benedetti, David Grohmann, Christine Poerschke)
 
+* SOLR-9055: Make collection backup/restore extensible. (Hrishikesh Gadre, Varun Thacker, Mark Miller)
+
 Optimizations
 ----------------------
 * SOLR-9704: Facet Module / JSON Facet API: Optimize blockChildren facets that have

solr/core/src/java/org/apache/solr/cloud/BackupCmd.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
 
+import org.apache.lucene.util.Version;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.cloud.ClusterState;
@@ -35,6 +36,7 @@ import org.apache.solr.common.cloud.Replica.State;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CoreAdminParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -68,31 +70,13 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
   public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
     String collectionName = message.getStr(COLLECTION_PROP);
     String backupName = message.getStr(NAME);
-    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
-    String asyncId = message.getStr(ASYNC);
     String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
 
-    String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
-    Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
-    if (commitName != null) {
-      SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
-      snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
-      if (!snapshotMeta.isPresent()) {
-        throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
-            + " does not exist for collection " + collectionName);
-      }
-      if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
-        throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
-            + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
-      }
-    }
-
-    Map<String, String> requestMap = new HashMap<>();
     Instant startTime = Instant.now();
 
     CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer();
     BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo));
-    BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader, collectionName);
+    BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader);
 
     // Backup location
     URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
@@ -106,50 +90,16 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     // Create a directory to store backup details.
     repository.createDirectory(backupPath);
 
-    log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
-        backupPath);
-
-    Collection<String> shardsToConsider = Collections.emptySet();
-    if (snapshotMeta.isPresent()) {
-      shardsToConsider = snapshotMeta.get().getShards();
-    }
-
-    for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
-      Replica replica = null;
-
-      if (snapshotMeta.isPresent()) {
-        if (!shardsToConsider.contains(slice.getName())) {
-          log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
-              slice.getName(), collectionName, snapshotMeta.get().getName());
-          continue;
-        }
-        replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
-      } else {
-        // Note - Actually this can return a null value when there is no leader for this shard.
-        replica = slice.getLeader();
-        if (replica == null) {
-          throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
-        }
+    String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
+    switch (strategy) {
+      case CollectionAdminParams.COPY_FILES_STRATEGY: {
+        copyIndexFiles(backupPath, message, results);
+        break;
       }
-
-      String coreName = replica.getStr(CORE_NAME_PROP);
-
-      ModifiableSolrParams params = new ModifiableSolrParams();
-      params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
-      params.set(NAME, slice.getName());
-      params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
-      params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
-      params.set(CORE_NAME_PROP, coreName);
-      if (snapshotMeta.isPresent()) {
-        params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
+      case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {
+        break;
       }
-
-      ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
-      log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
     }
-    log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);
-
-    ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap);
 
     log.info("Starting to backup ZK data for backupName={}", backupName);
 
@@ -168,6 +118,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
     properties.put(COLL_CONF, configName);
     properties.put(BackupManager.START_TIME_PROP, startTime.toString());
+    properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
     //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
     //if they are not the same then we can throw an error or have an 'overwriteConfig' flag
     //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
@@ -202,4 +153,73 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
 
     return r.get();
   }
+
+  private void copyIndexFiles(URI backupPath, ZkNodeProps request, NamedList results) throws Exception {
+    String collectionName = request.getStr(COLLECTION_PROP);
+    String backupName = request.getStr(NAME);
+    String asyncId = request.getStr(ASYNC);
+    String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
+    Map<String, String> requestMap = new HashMap<>();
+
+    String commitName = request.getStr(CoreAdminParams.COMMIT_NAME);
+    Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
+    if (commitName != null) {
+      SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
+      snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
+      if (!snapshotMeta.isPresent()) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
+            + " does not exist for collection " + collectionName);
+      }
+      if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
+            + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
+      }
+    }
+
+    log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
+        backupPath);
+
+    Collection<String> shardsToConsider = Collections.emptySet();
+    if (snapshotMeta.isPresent()) {
+      shardsToConsider = snapshotMeta.get().getShards();
+    }
+
+    for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
+      Replica replica = null;
+
+      if (snapshotMeta.isPresent()) {
+        if (!shardsToConsider.contains(slice.getName())) {
+          log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
+              slice.getName(), collectionName, snapshotMeta.get().getName());
+          continue;
+        }
+        replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
+      } else {
+        // Note - Actually this can return a null value when there is no leader for this shard.
+        replica = slice.getLeader();
+        if (replica == null) {
+          throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
+        }
+      }
+
+      String coreName = replica.getStr(CORE_NAME_PROP);
+
+      ModifiableSolrParams params = new ModifiableSolrParams();
+      params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
+      params.set(NAME, slice.getName());
+      params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
+      params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
+      params.set(CORE_NAME_PROP, coreName);
+      if (snapshotMeta.isPresent()) {
+        params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
+      }
+
+      ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
+      log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
+    }
+    log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);
+
+    ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap);
+  }
 }

solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java

@@ -87,7 +87,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
     URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
     URI backupPath = repository.resolve(location, backupName);
     ZkStateReader zkStateReader = ocmh.zkStateReader;
-    BackupManager backupMgr = new BackupManager(repository, zkStateReader, restoreCollectionName);
+    BackupManager backupMgr = new BackupManager(repository, zkStateReader);
 
     Properties properties = backupMgr.readBackupProperties(location, backupName);
     String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP);

solr/core/src/java/org/apache/solr/core/backup/BackupManager.java

@@ -68,7 +68,7 @@ public class BackupManager {
   protected final ZkStateReader zkStateReader;
   protected final BackupRepository repository;
 
-  public BackupManager(BackupRepository repository, ZkStateReader zkStateReader, String collectionName) {
+  public BackupManager(BackupRepository repository, ZkStateReader zkStateReader) {
     this.repository = Objects.requireNonNull(repository);
     this.zkStateReader = Objects.requireNonNull(zkStateReader);
   }
@@ -126,6 +126,7 @@ public class BackupManager {
    *
    * @param backupLoc The base path used to store the backup data.
    * @param backupId The unique name for the backup.
+   * @param collectionName The name of the collection whose meta-data is to be returned.
    * @return the meta-data information for the backed-up collection.
    * @throws IOException in case of errors.
    */

solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java

@@ -734,8 +734,14 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
         throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex);
       }
 
+      String strategy = req.getParams().get(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
+      if (!CollectionAdminParams.INDEX_BACKUP_STRATEGIES.contains(strategy)) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown index backup strategy " + strategy);
+      }
+
       Map<String, Object> params = req.getParams().getAll(null, NAME, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME);
       params.put(CoreAdminParams.BACKUP_LOCATION, location);
+      params.put(CollectionAdminParams.INDEX_BACKUP_STRATEGY, strategy);
       return params;
     }),
     RESTORE_OP(RESTORE, (req, rsp, h) -> {

solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java

@@ -38,6 +38,7 @@ import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.ImplicitDocRouter;
 import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CoreAdminParams;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -124,9 +125,24 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     }
 
     testBackupAndRestore(getCollectionName());
+    testConfigBackupOnly("conf1", getCollectionName());
     testInvalidPath(getCollectionName());
   }
 
+  /**
+   * This test validates the backup of collection configuration using
+   *  {@linkplain CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}.
+   *
+   * @param configName The config name for the collection to be backed up.
+   * @param collectionName The name of the collection to be backed up.
+   * @throws Exception in case of errors.
+   */
+  protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {
+    // This is deliberately no-op since we want to run this test only for one of the backup repository
+    // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository test
+    // implements this.
+  }
+
   // This test verifies the system behavior when the backup location cluster property is configured with an invalid
   // value for the specified repository (and the default backup location is not configured in solr.xml).
   private void testInvalidPath(String collectionName) throws Exception {

solr/core/src/test/org/apache/solr/cloud/TestHdfsCloudBackupRestore.java

@@ -16,10 +16,18 @@
  */
 package org.apache.solr.cloud;
 
+import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF;
+import static org.apache.solr.core.backup.BackupManager.*;
+
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import org.apache.commons.io.IOUtils;
@@ -28,7 +36,14 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.backup.BackupManager;
+import org.apache.solr.core.backup.repository.HdfsBackupRepository;
 import org.apache.solr.util.BadHdfsThreadsFilter;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -144,4 +159,45 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
   public String getBackupLocation() {
     return null;
   }
+
+  protected void testConfigBackupOnly(String configName, String collectionName) throws Exception {
+    String backupName = "configonlybackup";
+    CloudSolrClient solrClient = cluster.getSolrClient();
+
+    CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
+        .setRepositoryName(getBackupRepoName())
+        .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
+    backup.process(solrClient);
+
+    Map<String,String> params = new HashMap<>();
+    params.put("location", "/backup");
+    params.put("solr.hdfs.home", hdfsUri + "/solr");
+
+    HdfsBackupRepository repo = new HdfsBackupRepository();
+    repo.init(new NamedList<>(params));
+    BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader());
+
+    URI baseLoc = repo.createURI("/backup");
+
+    Properties props = mgr.readBackupProperties(baseLoc, backupName);
+    assertNotNull(props);
+    assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP));
+    assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP));
+    assertEquals(configName, props.getProperty(COLL_CONF));
+
+    DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName);
+    assertNotNull(collectionState);
+    assertEquals(collectionName, collectionState.getName());
+
+    URI configDirLoc = repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
+    assertTrue(repo.exists(configDirLoc));
+
+    Collection<String> expected = Arrays.asList(BACKUP_PROPS_FILE, ZK_STATE_DIR);
+    URI backupLoc = repo.resolve(baseLoc, backupName);
+    String[] dirs = repo.listAll(backupLoc);
+    for (String d : dirs) {
+      assertTrue(expected.contains(d));
+    }
+  }
+
 }

solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java

@@ -713,10 +713,12 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
     protected Optional<String> repositoryName = Optional.empty();
     protected String location;
     protected Optional<String> commitName = Optional.empty();
+    protected Optional<String> indexBackupStrategy = Optional.empty();
 
     public Backup(String collection, String name) {
       super(CollectionAction.BACKUP, collection);
       this.name = name;
+      this.repositoryName = Optional.empty();
     }
 
     @Override
@@ -760,6 +762,15 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
       return this;
     }
 
+    public Optional<String> getIndexBackupStrategy() {
+      return indexBackupStrategy;
+    }
+
+    public Backup setIndexBackupStrategy(String indexBackupStrategy) {
+      this.indexBackupStrategy = Optional.ofNullable(indexBackupStrategy);
+      return this;
+    }
+
     @Override
     public SolrParams getParams() {
       ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
@@ -772,6 +783,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
       if (commitName.isPresent()) {
         params.set(CoreAdminParams.COMMIT_NAME, commitName.get());
       }
+      if (indexBackupStrategy.isPresent()) {
+        params.set(CollectionAdminParams.INDEX_BACKUP_STRATEGY, indexBackupStrategy.get());
+      }
       return params;
     }
 

solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java

@@ -16,6 +16,9 @@
  */
 package org.apache.solr.common.params;
 
+import java.util.Arrays;
+import java.util.Collection;
+
 public interface CollectionAdminParams {
 
   /* Param used by DELETESTATUS call to clear all stored responses */
@@ -26,4 +29,25 @@ public interface CollectionAdminParams {
   String COUNT_PROP = "count";
 
 
+  /**
+   * A parameter to specify the name of the index backup strategy to be used.
+   */
+  public static final String INDEX_BACKUP_STRATEGY = "indexBackup";
+
+  /**
+   * This constant defines the index backup strategy based on copying index files to desired location.
+   */
+  public static final String COPY_FILES_STRATEGY = "copy-files";
+
+  /**
+   * This constant defines the strategy to not copy index files (useful for meta-data only backup).
+   */
+  public static final String NO_INDEX_BACKUP_STRATEGY = "none";
+
+  /**
+   * This constant defines a list of valid index backup strategies.
+   */
+  public static final Collection<String> INDEX_BACKUP_STRATEGIES =
+      Arrays.asList(COPY_FILES_STRATEGY, NO_INDEX_BACKUP_STRATEGY);
+
 }