diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a3bcaa8de93..29d60d0357c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -66,6 +66,11 @@ New Features * SOLR-9275: XML QueryParser support (defType=xmlparser) now extensible via configuration. (Christine Poerschke) +* SOLR-9038: Solr core snapshots: The current commit can be snapshotted which retains the commit and associates it with + a name. The core admin API can create snapshots, list them, and delete them. Snapshot names can be referenced in + doing a core backup, and in replication. Snapshot metadata is stored in a new snapshot_metadata/ dir. + (Hrishikesh Gadre via David Smiley) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java index 207c0e5d2a0..3616d4e545b 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java +++ b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java @@ -15,21 +15,26 @@ * limitations under the License. */ package org.apache.solr.core; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.update.SolrIndexWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - /** * A wrapper for an IndexDeletionPolicy instance. *

@@ -52,9 +57,11 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { private final Map reserves = new ConcurrentHashMap<>(); private volatile IndexCommit latestCommit; private final ConcurrentHashMap savedCommits = new ConcurrentHashMap<>(); + private final SolrSnapshotMetaDataManager snapshotMgr; - public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy) { + public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy, SolrSnapshotMetaDataManager snapshotMgr) { this.deletionPolicy = deletionPolicy; + this.snapshotMgr = snapshotMgr; } /** @@ -134,7 +141,6 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { } } - /** * Internal use for Lucene... do not explicitly call. */ @@ -185,7 +191,8 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { Long gen = delegate.getGeneration(); Long reserve = reserves.get(gen); if (reserve != null && System.nanoTime() < reserve) return; - if(savedCommits.containsKey(gen)) return; + if (savedCommits.containsKey(gen)) return; + if (snapshotMgr.isSnapshotted(gen)) return; delegate.delete(); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index faef1c8a4fd..2704e4ac868 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -81,6 +81,7 @@ import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.handler.IndexFetcher; import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.handler.RequestHandlerBase; @@ -184,6 +185,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { private final Map updateProcessorChains; private final Map infoRegistry; private final IndexDeletionPolicyWrapper solrDelPolicy; + private final SolrSnapshotMetaDataManager snapshotMgr; private final DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; private final Codec codec; @@ -414,7 +416,19 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } else { delPolicy = new SolrDeletionPolicy(); } - return new IndexDeletionPolicyWrapper(delPolicy); + + return new IndexDeletionPolicyWrapper(delPolicy, snapshotMgr); + } + + private SolrSnapshotMetaDataManager initSnapshotMetaDataManager() { + try { + String dirName = getDataDir() + SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR + "/"; + Directory snapshotDir = directoryFactory.get(dirName, DirContext.DEFAULT, + getSolrConfig().indexConfig.lockType); + return new SolrSnapshotMetaDataManager(this, snapshotDir); + } catch (IOException e) { + throw new IllegalStateException(e); + } } private void initListeners() { @@ -739,6 +753,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { initListeners(); + this.snapshotMgr = initSnapshotMetaDataManager(); this.solrDelPolicy = initDeletionPolicy(delPolicy); this.codec = initCodec(solrConfig, this.schema); @@ -1242,6 +1257,17 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } } + // Close the snapshots meta-data directory. + Directory snapshotsDir = snapshotMgr.getSnapshotsDir(); + try { + this.directoryFactory.release(snapshotsDir); + } catch (Throwable e) { + SolrException.log(log,e); + if (e instanceof Error) { + throw (Error) e; + } + } + if (coreStateClosed) { try { @@ -2343,6 +2369,14 @@ public final class SolrCore implements SolrInfoMBean, Closeable { return solrDelPolicy; } + /** + * @return A reference of {@linkplain SolrSnapshotMetaDataManager} + * managing the persistent snapshots for this Solr core. + */ + public SolrSnapshotMetaDataManager getSnapshotMetaDataManager() { + return snapshotMgr; + } + public ReentrantLock getRuleExpiryLock() { return ruleExpiryLock; } diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java new file mode 100644 index 00000000000..95df3fface2 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.Directory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class provides functionality required to handle the data files corresponding to Solr snapshots. + */ +public class SolrSnapshotManager { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number. + * + * @param dir The index directory storing the snapshot. + * @param gen The generation number for the {@linkplain IndexCommit} + * @throws IOException in case of I/O errors. + */ + public static void deleteIndexFiles ( Directory dir, Collection snapshots, long gen ) throws IOException { + List commits = DirectoryReader.listCommits(dir); + Map refCounts = buildRefCounts(snapshots, commits); + for (IndexCommit ic : commits) { + if (ic.getGeneration() == gen) { + deleteIndexFiles(dir,refCounts, ic); + break; + } + } + } + + /** + * This method deletes all files not corresponding to a configured snapshot in the specified index directory. + * + * @param dir The index directory to search for. + * @throws IOException in case of I/O errors. + */ + public static void deleteNonSnapshotIndexFiles (Directory dir, Collection snapshots) throws IOException { + List commits = DirectoryReader.listCommits(dir); + Map refCounts = buildRefCounts(snapshots, commits); + Set snapshotGenNumbers = snapshots.stream() + .map(SnapshotMetaData::getGenerationNumber) + .collect(Collectors.toSet()); + for (IndexCommit ic : commits) { + if (!snapshotGenNumbers.contains(ic.getGeneration())) { + deleteIndexFiles(dir,refCounts, ic); + } + } + } + + /** + * This method computes reference count for the index files by taking into consideration + * (a) configured snapshots and (b) files sharing between two or more {@linkplain IndexCommit} instances. + * + * @param snapshots A collection of user configured snapshots + * @param commits A list of {@linkplain IndexCommit} instances + * @return A map containing reference count for each index file referred in one of the {@linkplain IndexCommit} instances. + * @throws IOException in case of I/O error. + */ + @VisibleForTesting + static Map buildRefCounts (Collection snapshots, List commits) throws IOException { + Map result = new HashMap<>(); + Map commitsByGen = commits.stream().collect( + Collectors.toMap(IndexCommit::getGeneration, Function.identity())); + + for(SnapshotMetaData md : snapshots) { + IndexCommit ic = commitsByGen.get(md.getGenerationNumber()); + if (ic != null) { + Collection fileNames = ic.getFileNames(); + for(String fileName : fileNames) { + int refCount = result.getOrDefault(fileName, 0); + result.put(fileName, refCount+1); + } + } + } + + return result; + } + + /** + * This method deletes the index files associated with specified indexCommit provided they + * are not referred by some other {@linkplain IndexCommit}. + * + * @param dir The index directory containing the {@linkplain IndexCommit} to be deleted. + * @param refCounts A map containing reference counts for each file associated with every {@linkplain IndexCommit} + * in the specified directory. + * @param indexCommit The {@linkplain IndexCommit} whose files need to be deleted. + * @throws IOException in case of I/O errors. + */ + private static void deleteIndexFiles ( Directory dir, Map refCounts, IndexCommit indexCommit ) throws IOException { + log.info("Deleting index files for index commit with generation {} in directory {}", indexCommit.getGeneration(), dir); + for (String fileName : indexCommit.getFileNames()) { + try { + // Ensure that a file being deleted is not referred by some other commit. + int ref = refCounts.getOrDefault(fileName, 0); + log.debug("Reference count for file {} is {}", fileName, ref); + if (ref == 0) { + dir.deleteFile(fileName); + } + } catch (IOException e) { + log.warn("Unable to delete file {} in directory {} due to exception {}", fileName, dir, e.getMessage()); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java new file mode 100644 index 00000000000..26cbe215f70 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.google.common.base.Preconditions; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is responsible to manage the persistent snapshots meta-data for the Solr indexes. The + * persistent snapshots are implemented by relying on Lucene {@linkplain IndexDeletionPolicy} + * abstraction to configure a specific {@linkplain IndexCommit} to be retained. The + * {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to create/delete the Solr index + * snapshots. + */ +public class SolrSnapshotMetaDataManager { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata"; + + /** + * A class defining the meta-data for a specific snapshot. + */ + public static class SnapshotMetaData { + private String name; + private String indexDirPath; + private long generationNumber; + + public SnapshotMetaData(String name, String indexDirPath, long generationNumber) { + super(); + this.name = name; + this.indexDirPath = indexDirPath; + this.generationNumber = generationNumber; + } + + public String getName() { + return name; + } + + public String getIndexDirPath() { + return indexDirPath; + } + + public long getGenerationNumber() { + return generationNumber; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("SnapshotMetaData[name="); + builder.append(name); + builder.append(", indexDirPath="); + builder.append(indexDirPath); + builder.append(", generation="); + builder.append(generationNumber); + builder.append("]"); + return builder.toString(); + } + } + + /** Prefix used for the save file. */ + public static final String SNAPSHOTS_PREFIX = "snapshots_"; + private static final int VERSION_START = 0; + private static final int VERSION_CURRENT = VERSION_START; + private static final String CODEC_NAME = "solr-snapshots"; + + // The index writer which maintains the snapshots metadata + private long nextWriteGen; + + private final Directory dir; + + /** Used to map snapshot name to snapshot meta-data. */ + protected final Map nameToDetailsMapping = new LinkedHashMap<>(); + /** Used to figure out the *current* index data directory path */ + private final SolrCore solrCore; + + /** + * A constructor. + * + * @param dir The directory where the snapshot meta-data should be stored. Enables updating + * the existing meta-data. + * @throws IOException in case of errors. + */ + public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir) throws IOException { + this(solrCore, dir, OpenMode.CREATE_OR_APPEND); + } + + /** + * A constructor. + * + * @param dir The directory where the snapshot meta-data is stored. + * @param mode CREATE If previous meta-data should be erased. + * APPEND If previous meta-data should be read and updated. + * CREATE_OR_APPEND Creates a new meta-data structure if one does not exist + * Updates the existing structure if one exists. + * @throws IOException in case of errors. + */ + public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir, OpenMode mode) throws IOException { + this.solrCore = solrCore; + this.dir = dir; + + if (mode == OpenMode.CREATE) { + deleteSnapshotMetadataFiles(); + } + + loadFromSnapshotMetadataFile(); + + if (mode == OpenMode.APPEND && nextWriteGen == 0) { + throw new IllegalStateException("no snapshots stored in this directory"); + } + } + + /** + * @return The snapshot meta-data directory + */ + public Directory getSnapshotsDir() { + return dir; + } + + /** + * This method creates a new snapshot meta-data entry. + * + * @param name The name of the snapshot. + * @param indexDirPath The directory path where the index files are stored. + * @param gen The generation number for the {@linkplain IndexCommit} being snapshotted. + * @throws IOException in case of I/O errors. + */ + public synchronized void snapshot(String name, String indexDirPath, long gen) throws IOException { + Preconditions.checkNotNull(name); + + log.info("Creating the snapshot named {} for core {} associated with index commit with generation {} in directory {}" + , name, solrCore.getName(), gen, indexDirPath); + + if(nameToDetailsMapping.containsKey(name)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name " + name + " already exists"); + } + + SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen); + nameToDetailsMapping.put(name, d); + + boolean success = false; + try { + persist(); + success = true; + } finally { + if (!success) { + try { + release(name); + } catch (Exception e) { + // Suppress so we keep throwing original exception + } + } + } + } + + /** + * This method deletes a previously created snapshot (if any). + * + * @param name The name of the snapshot to be deleted. + * @return The snapshot meta-data if the snapshot with the snapshot name exists. + * @throws IOException in case of I/O error + */ + public synchronized Optional release(String name) throws IOException { + log.info("Deleting the snapshot named {} for core {}", name, solrCore.getName()); + SnapshotMetaData result = nameToDetailsMapping.remove(Preconditions.checkNotNull(name)); + if(result != null) { + boolean success = false; + try { + persist(); + success = true; + } finally { + if (!success) { + nameToDetailsMapping.put(name, result); + } + } + } + return Optional.ofNullable(result); + } + + /** + * This method returns if snapshot is created for the specified generation number in + * the *current* index directory. + * + * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked. + * @return true if the snapshot is created. + * false otherwise. + */ + public synchronized boolean isSnapshotted(long genNumber) { + return !nameToDetailsMapping.isEmpty() && isSnapshotted(solrCore.getIndexDir(), genNumber); + } + + /** + * This method returns if snapshot is created for the specified generation number in + * the specified index directory. + * + * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked. + * @return true if the snapshot is created. + * false otherwise. + */ + public synchronized boolean isSnapshotted(String indexDirPath, long genNumber) { + return !nameToDetailsMapping.isEmpty() + && nameToDetailsMapping.values().stream() + .anyMatch(entry -> entry.getIndexDirPath().equals(indexDirPath) && entry.getGenerationNumber() == genNumber); + } + + /** + * This method returns the snapshot meta-data for the specified name (if it exists). + * + * @param name The name of the snapshot + * @return The snapshot meta-data if exists. + */ + public synchronized Optional getSnapshotMetaData(String name) { + return Optional.ofNullable(nameToDetailsMapping.get(name)); + } + + /** + * @return A list of snapshots created so far. + */ + public synchronized List listSnapshots() { + // We create a copy for thread safety. + return new ArrayList<>(nameToDetailsMapping.keySet()); + } + + /** + * This method returns a list of snapshots created in a specified index directory. + * + * @param indexDirPath The index directory path. + * @return a list snapshots stored in the specified directory. + */ + public synchronized Collection listSnapshotsInIndexDir(String indexDirPath) { + return nameToDetailsMapping.values().stream() + .filter(entry -> indexDirPath.equals(entry.getIndexDirPath())) + .collect(Collectors.toList()); + } + + /** + * This method returns the {@linkplain IndexCommit} associated with the specified + * commitName. A snapshot with specified commitName must + * be created before invoking this method. + * + * @param commitName The name of persisted commit + * @return the {@linkplain IndexCommit} + * @throws IOException in case of I/O error. + */ + public Optional getIndexCommitByName(String commitName) throws IOException { + Optional result = Optional.empty(); + Optional metaData = getSnapshotMetaData(commitName); + if (metaData.isPresent()) { + String indexDirPath = metaData.get().getIndexDirPath(); + long gen = metaData.get().getGenerationNumber(); + + Directory d = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE); + try { + result = DirectoryReader.listCommits(d) + .stream() + .filter(ic -> ic.getGeneration() == gen) + .findAny(); + + if (!result.isPresent()) { + log.warn("Unable to find commit with generation {} in the directory {}", gen, indexDirPath); + } + + } finally { + solrCore.getDirectoryFactory().release(d); + } + } else { + log.warn("Commit with name {} is not persisted for core {}", commitName, solrCore.getName()); + } + + return result; + } + + private synchronized void persist() throws IOException { + String fileName = SNAPSHOTS_PREFIX + nextWriteGen; + IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); + boolean success = false; + try { + CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + out.writeVInt(nameToDetailsMapping.size()); + for(Entry ent : nameToDetailsMapping.entrySet()) { + out.writeString(ent.getKey()); + out.writeString(ent.getValue().getIndexDirPath()); + out.writeVLong(ent.getValue().getGenerationNumber()); + } + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + IOUtils.deleteFilesIgnoringExceptions(dir, fileName); + } else { + IOUtils.close(out); + } + } + + dir.sync(Collections.singletonList(fileName)); + + if (nextWriteGen > 0) { + String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); + // exception OK: likely it didn't exist + IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); + } + + nextWriteGen++; + } + + private synchronized void deleteSnapshotMetadataFiles() throws IOException { + for(String file : dir.listAll()) { + if (file.startsWith(SNAPSHOTS_PREFIX)) { + dir.deleteFile(file); + } + } + } + + /** + * Reads the snapshot meta-data information from the given {@link Directory}. + */ + private synchronized void loadFromSnapshotMetadataFile() throws IOException { + log.info("Loading from snapshot metadata file..."); + long genLoaded = -1; + IOException ioe = null; + List snapshotFiles = new ArrayList<>(); + for(String file : dir.listAll()) { + if (file.startsWith(SNAPSHOTS_PREFIX)) { + long gen = Long.parseLong(file.substring(SNAPSHOTS_PREFIX.length())); + if (genLoaded == -1 || gen > genLoaded) { + snapshotFiles.add(file); + Map snapshotMetaDataMapping = new HashMap<>(); + IndexInput in = dir.openInput(file, IOContext.DEFAULT); + try { + CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); + int count = in.readVInt(); + for(int i=0;i 1) { + // Remove any broken / old snapshot files: + String curFileName = SNAPSHOTS_PREFIX + genLoaded; + for(String file : snapshotFiles) { + if (!curFileName.equals(file)) { + IOUtils.deleteFilesIgnoringExceptions(dir, file); + } + } + } + nextWriteGen = 1+genLoaded; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java b/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java new file mode 100644 index 00000000000..3242cd347f1 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java @@ -0,0 +1,22 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +/** + * Core classes for Solr's persistent snapshots functionality + */ +package org.apache.solr.core.snapshots; \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index cfbd42de863..0612f2f1a35 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -81,6 +81,9 @@ import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.handler.ReplicationHandler.*; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -454,9 +457,18 @@ public class IndexFetcher { // let the system know we are changing dir's and the old one // may be closed if (indexDir != null) { - LOG.info("removing old index directory " + indexDir); solrCore.getDirectoryFactory().doneWithDirectory(indexDir); - solrCore.getDirectoryFactory().remove(indexDir); + + SolrSnapshotMetaDataManager snapshotsMgr = solrCore.getSnapshotMetaDataManager(); + Collection snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath); + + // Delete the old index directory only if no snapshot exists in that directory. + if(snapshots.isEmpty()) { + LOG.info("removing old index directory " + indexDir); + solrCore.getDirectoryFactory().remove(indexDir); + } else { + SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots); + } } } diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 2e889b72f0d..e473a63b1ba 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -87,6 +87,7 @@ import org.apache.solr.core.SolrDeletionPolicy; import org.apache.solr.core.SolrEventListener; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.SolrIndexSearcher; @@ -505,11 +506,24 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw numberToKeep = Integer.MAX_VALUE; } - IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); - IndexCommit indexCommit = delPolicy.getLatestCommit(); + IndexCommit indexCommit = null; + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + indexCommit = commit.get(); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName + + " for core " + core.getName()); + } + } else { + IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); + indexCommit = delPolicy.getLatestCommit(); - if (indexCommit == null) { - indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + if (indexCommit == null) { + indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + } } String location = params.get(CoreAdminParams.BACKUP_LOCATION); @@ -532,7 +546,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } // small race here before the commit point is saved - SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME)); + SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME), commitName); snapShooter.validateCreateSnapshot(); snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl); diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java index d3c98fac432..6aef35c448a 100644 --- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java @@ -19,6 +19,7 @@ package org.apache.solr.handler; import java.lang.invoke.MethodHandles; import java.net.URI; import java.text.SimpleDateFormat; +import java.util.Collection; import java.util.Date; import java.util.Locale; import java.util.concurrent.Callable; @@ -32,6 +33,9 @@ import org.apache.solr.common.SolrException; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +67,7 @@ public class RestoreCore implements Callable { String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; + String indexDirPath = core.getIndexDir(); Directory restoreIndexDir = null; Directory indexDir = null; try { @@ -71,7 +76,7 @@ public class RestoreCore implements Callable { DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Prefer local copy. - indexDir = core.getDirectoryFactory().get(core.getIndexDir(), + indexDir = core.getDirectoryFactory().get(indexDirPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Move all files from backupDir to restoreIndexDir @@ -130,7 +135,16 @@ public class RestoreCore implements Callable { } if (success) { core.getDirectoryFactory().doneWithDirectory(indexDir); - core.getDirectoryFactory().remove(indexDir); + + SolrSnapshotMetaDataManager snapshotsMgr = core.getSnapshotMetaDataManager(); + Collection snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath); + + // Delete the old index directory only if no snapshot exists in that directory. + if (snapshots.isEmpty()) { + core.getDirectoryFactory().remove(indexDir); + } else { + SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots); + } } return true; diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index 5ac324329a0..e12649dcd91 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -26,12 +26,14 @@ import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.function.Consumer; import com.google.common.base.Preconditions; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; @@ -39,6 +41,7 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.BackupRepository.PathType; import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; import org.slf4j.Logger; @@ -59,6 +62,7 @@ public class SnapShooter { private URI baseSnapDirPath = null; private URI snapshotDirPath = null; private BackupRepository backupRepo = null; + private String commitName; // can be null @Deprecated public SnapShooter(SolrCore core, String location, String snapshotName) { @@ -71,14 +75,14 @@ public class SnapShooter { } else { snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); } - initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName); + initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName, null); } - public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) { - initialize(backupRepo, core, location, snapshotName); + public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) { + initialize(backupRepo, core, location, snapshotName, commitName); } - private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) { + private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) { this.solrCore = Preconditions.checkNotNull(core); this.backupRepo = Preconditions.checkNotNull(backupRepo); this.baseSnapDirPath = backupRepo.createURI(Preconditions.checkNotNull(location)).normalize(); @@ -90,6 +94,7 @@ public class SnapShooter { directoryName = "snapshot." + fmt.format(new Date()); } this.snapshotDirPath = backupRepo.createURI(location, directoryName); + this.commitName = commitName; } public BackupRepository getBackupRepository() { @@ -145,16 +150,26 @@ public class SnapShooter { } public NamedList createSnapshot() throws Exception { - IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); RefCounted searcher = solrCore.getSearcher(); try { - //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? - IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); - deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); - try { - return createSnapshot(indexCommit); - } finally { - deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + return createSnapshot(commit.get()); + } + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to find an index commit with name " + commitName + + " for core " + solrCore.getName()); + } else { + //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? + IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); + IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); + deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); + try { + return createSnapshot(indexCommit); + } finally { + deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + } } } finally { searcher.decref(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 9b9aafa6ae3..e4103c54e17 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -34,6 +34,7 @@ import java.util.concurrent.Future; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; @@ -59,9 +60,13 @@ import org.apache.solr.core.CachingDirectoryFactory; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.handler.RestoreCore; import org.apache.solr.handler.SnapShooter; import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp; @@ -794,22 +799,26 @@ enum CoreAdminOperation implements CoreAdminOp { + " parameter or as a default repository property"); } - try (SolrCore core = it.handler.coreContainer.getCore(cname)) { - SnapShooter snapShooter = new SnapShooter(repository, core, location, name); - // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. - // But we want to throw. One reason is that - // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared - // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. - if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation()); + // An optional parameter to describe the snapshot to be backed-up. If this + // parameter is not supplied, the latest index commit is backed-up. + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + + try (SolrCore core = it.handler.coreContainer.getCore(cname)) { + SnapShooter snapShooter = new SnapShooter(repository, core, location, name, commitName); + // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. + // But we want to throw. One reason is that + // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared + // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. + if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation()); + } + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapshot(); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to backup core=" + cname + " because " + e, e); } - snapShooter.validateCreateSnapshot(); - snapShooter.createSnapshot(); - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Failed to backup core=" + cname + " because " + e, e); - } }), RESTORECORE_OP(RESTORECORE, it -> { @@ -845,6 +854,92 @@ enum CoreAdminOperation implements CoreAdminOp { throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); } } + }), + CREATESNAPSHOT_OP(CREATESNAPSHOT, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); + String cname = params.required().get(CoreAdminParams.CORE); + try (SolrCore core = cc.getCore(cname)) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + String indexDirPath = core.getIndexDir(); + IndexCommit ic = core.getDeletionPolicy().getLatestCommit(); + if (ic == null) { + RefCounted searcher = core.getSearcher(); + try { + ic = searcher.get().getIndexReader().getIndexCommit(); + } finally { + searcher.decref(); + } + } + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + mgr.snapshot(commitName, indexDirPath, ic.getGeneration()); + + it.rsp.add("core", core.getName()); + it.rsp.add("commitName", commitName); + it.rsp.add("indexDirPath", indexDirPath); + it.rsp.add("generation", ic.getGeneration()); + } + }), + DELETESNAPSHOT_OP(DELETESNAPSHOT, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); + String cname = params.required().get(CoreAdminParams.CORE); + try (SolrCore core = cc.getCore(cname)) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + Optional metadata = mgr.release(commitName); + if (metadata.isPresent()) { + long gen = metadata.get().getGenerationNumber(); + String indexDirPath = metadata.get().getIndexDirPath(); + + // If the directory storing the snapshot is not the same as the *current* core + // index directory, then delete the files corresponding to this snapshot. + // Otherwise we leave the index files related to snapshot as is (assuming the + // underlying Solr IndexDeletionPolicy will clean them up appropriately). + if (!indexDirPath.equals(core.getIndexDir())) { + Directory d = core.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE); + try { + SolrSnapshotManager.deleteIndexFiles(d, mgr.listSnapshotsInIndexDir(indexDirPath), gen); + } finally { + core.getDirectoryFactory().release(d); + } + } + } + } + }), + LISTSNAPSHOTS_OP(LISTSNAPSHOTS, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String cname = params.required().get(CoreAdminParams.CORE); + try ( SolrCore core = cc.getCore(cname) ) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + NamedList result = new NamedList(); + for (String name : mgr.listSnapshots()) { + Optional metadata = mgr.getSnapshotMetaData(name); + if ( metadata.isPresent() ) { + NamedList props = new NamedList<>(); + props.add("generation", String.valueOf(metadata.get().getGenerationNumber())); + props.add("indexDirPath", metadata.get().getIndexDirPath()); + result.add(name, props); + } + } + it.rsp.add("snapshots", result); + } }); final CoreAdminParams.CoreAdminAction action; diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java new file mode 100644 index 00000000000..aacac5242cb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexNotFoundException; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.lucene.util.TestUtil; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot; +import org.apache.solr.client.solrj.request.CoreAdminRequest.DeleteSnapshot; +import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; +import org.apache.solr.handler.BackupRestoreUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; + +@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test +@Slow +public class TestSolrCoreSnapshots extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static long docsSeed; // see indexDocs() + + @BeforeClass + public static void setupClass() throws Exception { + useFactory("solr.StandardDirectoryFactory"); + configureCluster(1)// nodes + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + docsSeed = random().nextLong(); + } + + @AfterClass + public static void teardownClass() throws Exception { + System.clearProperty("test.build.data"); + System.clearProperty("test.cache.data"); + } + + @Test + public void testBackupRestore() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + String location = createTempDir().toFile().getAbsolutePath(); + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String replicaBaseUrl = replica.getStr(BASE_URL_PROP); + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String backupName = TestUtil.randomSimpleString(random(), 1, 5); + String commitName = TestUtil.randomSimpleString(random(), 1, 5); + String duplicateName = commitName.concat("_duplicate"); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName); + // Create another snapshot referring to the same index commit to verify the + // reference counting implementation during snapshot deletion. + SnapshotMetaData duplicateCommit = createSnapshot(adminClient, coreName, duplicateName); + + assertEquals (metaData.getIndexDirPath(), duplicateCommit.getIndexDirPath()); + assertEquals (metaData.getGenerationNumber(), duplicateCommit.getGenerationNumber()); + + // Delete all documents + masterClient.deleteByQuery("*:*"); + masterClient.commit(); + BackupRestoreUtils.verifyDocs(0, cluster.getSolrClient(), collectionName); + + // Verify that the index directory contains at least 2 index commits - one referred by the snapshots + // and the other containing document deletions. + { + List commits = listCommits(metaData.getIndexDirPath()); + assertTrue(2 <= commits.size()); + } + + // Backup the earlier created snapshot. + { + Map params = new HashMap<>(); + params.put("name", backupName); + params.put("commitName", commitName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); + } + + // Restore the backup + { + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); + BackupRestoreUtils.verifyDocs(nDocs, cluster.getSolrClient(), collectionName); + } + + // Verify that the old index directory (before restore) contains only those index commits referred by snapshots. + { + List commits = listCommits(metaData.getIndexDirPath()); + assertEquals(1, commits.size()); + assertEquals(metaData.getGenerationNumber(), commits.get(0).getGeneration()); + } + + // Delete first snapshot + deleteSnapshot(adminClient, coreName, commitName); + + // Verify that corresponding index files have NOT been deleted (due to reference counting). + assertFalse(listCommits(metaData.getIndexDirPath()).isEmpty()); + + // Delete second snapshot + deleteSnapshot(adminClient, coreName, duplicateCommit.getName()); + + // Verify that corresponding index files have been deleted. + assertTrue(listCommits(duplicateCommit.getIndexDirPath()).isEmpty()); + } + } + + @Test + public void testHandlingSharedIndexFiles() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots_IndexFileSharing"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String replicaBaseUrl = replica.getStr(BASE_URL_PROP); + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String backupName = TestUtil.randomSimpleString(random(), 1, 5); + String location = createTempDir().toFile().getAbsolutePath(); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + int numTests = TestUtil.nextInt(random(), 2, 5); + List snapshots = new ArrayList<>(numTests); + + // Create multiple commits and create a snapshot per commit. + // This should result in Lucene reusing some of the segments for later index commits. + for (int attempt=0; attempt 0) { + //Delete a few docs + int numDeletes = TestUtil.nextInt(random(), 1, nDocs); + for(int i=0; i params = new HashMap<>(); + params.put("name", backupName); + params.put("commitName", snapshots.get(0).getName()); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); + } + + // Restore the backup. The purpose of the restore operation is to change the *current* index directory. + // This is required since we delegate the file deletion to underlying IndexDeletionPolicy in case of + // *current* index directory. Hence for the purpose of this test, we want to ensure that the created + // snapshots are NOT in the *current* index directory. + { + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); + } + + { + SnapshotMetaData snapshotMetaData = snapshots.get(0); + + List commits = listCommits(snapshotMetaData.getIndexDirPath()); + // Check if number of index commits are > 0 to ensure index file sharing. + assertTrue(commits.size() > 0); + Map refCounts = SolrSnapshotManager.buildRefCounts(snapshots, commits); + + Optional ic = commits.stream() + .filter(entry -> entry.getGeneration() == snapshotMetaData.getGenerationNumber()) + .findFirst(); + assertTrue(ic.isPresent()); + Collection nonSharedFiles = new ArrayList<>(); + Collection sharedFiles = new ArrayList<>(); + for (String fileName : ic.get().getFileNames()) { + if (refCounts.getOrDefault(fileName, 0) > 1) { + sharedFiles.add(fileName); + } else { + nonSharedFiles.add(fileName); + } + } + + // Delete snapshot + deleteSnapshot(adminClient, coreName, snapshotMetaData.getName()); + + // Verify that the shared files are not deleted. + for (String fileName : sharedFiles) { + Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName); + assertTrue(path + " should exist.", Files.exists(path)); + } + + // Verify that the non-shared files are deleted. + for (String fileName : nonSharedFiles) { + Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName); + assertFalse(path + " should not exist.", Files.exists(path)); + } + } + } + } + + @Test + public void testIndexOptimization() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots_IndexOptimization"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String commitName = TestUtil.randomSimpleString(random(), 1, 5); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName); + + int numTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1; + for (int attempt=0; attempt 0) { + //Delete a few docs + int numDeletes = TestUtil.nextInt(random(), 1, nDocs); + for(int i=0; i result = new ArrayList<>(apiResult.size()); + for(int i = 0 ; i < apiResult.size(); i++) { + String commitName = apiResult.getName(i); + String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get("indexDirPath"); + long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get("generation")); + result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber)); + } + return result; + } + + private List listCommits(String directory) throws Exception { + SimpleFSDirectory dir = new SimpleFSDirectory(Paths.get(directory)); + try { + return DirectoryReader.listCommits(dir); + } catch (IndexNotFoundException ex) { + // This can happen when the delete snapshot functionality cleans up the index files (when the directory + // storing these files is not the *current* index directory). + return Collections.emptyList(); + } + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java b/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java index e2f4304ac82..34509cf8b3e 100644 --- a/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java +++ b/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java @@ -18,11 +18,15 @@ package org.apache.solr.handler; import java.io.IOException; +import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URL; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; +import org.apache.commons.io.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; @@ -64,4 +68,37 @@ public class BackupRestoreUtils extends LuceneTestCase { assertEquals(0, response.getStatus()); assertEquals(nDocs, response.getResults().getNumFound()); } + + public static void runCoreAdminCommand(String baseUrl, String coreName, String action, Map params) throws IOException { + StringBuilder builder = new StringBuilder(); + builder.append(baseUrl); + builder.append("/admin/cores?action="); + builder.append(action); + builder.append("&core="); + builder.append(coreName); + for (Map.Entry p : params.entrySet()) { + builder.append("&"); + builder.append(p.getKey()); + builder.append("="); + builder.append(p.getValue()); + } + String masterUrl = builder.toString(); + executeHttpRequest(masterUrl); + } + + public static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { + String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName; + executeHttpRequest(masterUrl); + } + + static void executeHttpRequest(String requestUrl) throws IOException { + InputStream stream = null; + try { + URL url = new URL(requestUrl); + stream = url.openStream(); + stream.close(); + } finally { + IOUtils.closeQuietly(stream); + } + } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java index a84042833ab..4e8d4ccd584 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java +++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java @@ -18,11 +18,11 @@ package org.apache.solr.handler; import java.io.IOException; -import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; +import java.util.HashMap; +import java.util.Map; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.commons.io.IOUtils; @@ -44,6 +44,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; @@ -176,16 +177,19 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) { // Create a backup. if (testViaReplicationHandler) { - log.info("Running Backup/restore via replication handler"); - runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName); + log.info("Running Backup via replication handler"); + BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName); CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null); while (!checkBackupStatus.success) { checkBackupStatus.fetchStatus(); Thread.sleep(1000); } } else { - log.info("Running Backup/restore via core admin api"); - runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), "hdfs", backupName); + log.info("Running Backup via core admin api"); + Map params = new HashMap<>(); + params.put("name", backupName); + params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs"); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); } int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1; @@ -214,38 +218,22 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { } // Snapshooter prefixes "snapshot." to the backup name. if (testViaReplicationHandler) { + log.info("Running Restore via replication handler"); // Snapshooter prefixes "snapshot." to the backup name. - runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName); + BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName); while (!TestRestoreCore.fetchRestoreStatus(baseUrl, coreName)) { Thread.sleep(1000); } } else { - runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), "hdfs", "snapshot." + backupName); + log.info("Running Restore via core admin api"); + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs"); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); } //See if restore was successful by checking if all the docs are present again BackupRestoreUtils.verifyDocs(nDocs, masterClient, coreName); } } } - - static void runCoreAdminCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { - String masterUrl = baseUrl + "/admin/cores?action=" + action + "&core="+coreName+"&repository="+repoName+"&name="+backupName; - executeHttpRequest(masterUrl); - } - - static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { - String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName; - executeHttpRequest(masterUrl); - } - - static void executeHttpRequest(String requestUrl) throws IOException { - InputStream stream = null; - try { - URL url = new URL(requestUrl); - stream = url.openStream(); - stream.close(); - } finally { - IOUtils.closeQuietly(stream); - } - } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index 60b601e374f..006b2e137d8 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -20,6 +20,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -66,6 +67,7 @@ import org.apache.solr.core.CachingDirectoryFactory; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.core.StandardDirectoryFactory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.util.FileUtils; import org.junit.After; import org.junit.Before; @@ -898,8 +900,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { CachingDirectoryFactory dirFactory = (CachingDirectoryFactory) core.getDirectoryFactory(); synchronized (dirFactory) { Set livePaths = dirFactory.getLivePaths(); - // one for data, one for hte index under data - assertEquals(livePaths.toString(), 2, livePaths.size()); + // one for data, one for hte index under data and one for the snapshot metadata. + assertEquals(livePaths.toString(), 3, livePaths.size()); // :TODO: assert that one of the paths is a subpath of hte other } if (dirFactory instanceof StandardDirectoryFactory) { @@ -910,14 +912,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { } private int indexDirCount(String ddir) { - String[] list = new File(ddir).list(); - int cnt = 0; - for (String file : list) { - if (!file.endsWith(".properties")) { - cnt++; + String[] list = new File(ddir).list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + File f = new File(dir, name); + return f.isDirectory() && !SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR.equals(name); } - } - return cnt; + }); + return list.length; } private void pullFromMasterToSlave() throws MalformedURLException, diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java index 7d9e356e73f..f3e4e199f83 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java @@ -453,6 +453,63 @@ public class CoreAdminRequest extends SolrRequest { } + public static class CreateSnapshot extends CoreAdminRequest { + private String commitName; + + public CreateSnapshot(String commitName) { + super(); + this.action = CoreAdminAction.CREATESNAPSHOT; + if(commitName == null) { + throw new NullPointerException("Please specify non null value for commitName parameter."); + } + this.commitName = commitName; + } + + public String getCommitName() { + return commitName; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(CoreAdminParams.COMMIT_NAME, this.commitName); + return params; + } + } + + public static class DeleteSnapshot extends CoreAdminRequest { + private String commitName; + + public DeleteSnapshot(String commitName) { + super(); + this.action = CoreAdminAction.DELETESNAPSHOT; + + if(commitName == null) { + throw new NullPointerException("Please specify non null value for commitName parameter."); + } + this.commitName = commitName; + } + + public String getCommitName() { + return commitName; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(CoreAdminParams.COMMIT_NAME, this.commitName); + return params; + } + } + + public static class ListSnapshots extends CoreAdminRequest { + public ListSnapshots() { + super(); + this.action = CoreAdminAction.LISTSNAPSHOTS; + } + } + + public CoreAdminRequest() { super( METHOD.GET, "/admin/cores" ); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 7455cbf1071..7f90a90bee1 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -118,6 +118,11 @@ public abstract class CoreAdminParams */ public static final String BACKUP_LOCATION = "location"; + /** + * A parameter to specify the name of the commit to be stored during the backup operation. + */ + public static final String COMMIT_NAME = "commitName"; + public enum CoreAdminAction { STATUS(true), UNLOAD, @@ -141,7 +146,10 @@ public abstract class CoreAdminParams INVOKE, //Internal APIs to backup and restore a core BACKUPCORE, - RESTORECORE; + RESTORECORE, + CREATESNAPSHOT, + DELETESNAPSHOT, + LISTSNAPSHOTS; public final boolean isRead;