Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9269: Snapshots (Solr core level)

This commit is contained in:
David Smiley 2016-07-28 14:23:09 -04:00
parent 7bf019a9c2
commit b7aa0b56be
17 changed files with 1365 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -103,6 +103,11 @@ New Features
* SOLR-9200: Add Delegation Token Support to Solr.
(Gregory Chanan)
* SOLR-9038: Solr core snapshots: The current commit can be snapshotted which retains the commit and associates it with
a name. The core admin API can create snapshots, list them, and delete them. Snapshot names can be referenced in
doing a core backup, and in replication. Snapshot metadata is stored in a new snapshot_metadata/ dir.
(Hrishikesh Gadre via David Smiley)
Bug Fixes
----------------------

27
solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
View File

@ -15,21 +15,26 @@
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.update.SolrIndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* A wrapper for an IndexDeletionPolicy instance.
* <p>
@ -52,9 +57,11 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
private final Map<Long, Long> reserves = new ConcurrentHashMap<>();
private volatile IndexCommit latestCommit;
private final ConcurrentHashMap<Long, AtomicInteger> savedCommits = new ConcurrentHashMap<>();
private final SolrSnapshotMetaDataManager snapshotMgr;
public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy) {
public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy, SolrSnapshotMetaDataManager snapshotMgr) {
this.deletionPolicy = deletionPolicy;
this.snapshotMgr = snapshotMgr;
}
/**
@ -134,7 +141,6 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
}
}
/**
* Internal use for Lucene... do not explicitly call.
*/
@ -185,7 +191,8 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
Long gen = delegate.getGeneration();
Long reserve = reserves.get(gen);
if (reserve != null && System.nanoTime() < reserve) return;
if(savedCommits.containsKey(gen)) return;
if (savedCommits.containsKey(gen)) return;
if (snapshotMgr.isSnapshotted(gen)) return;
delegate.delete();
}

36
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -81,6 +81,7 @@ import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.handler.IndexFetcher;
import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.handler.RequestHandlerBase;
@ -184,6 +185,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private final Map<String,UpdateRequestProcessorChain> updateProcessorChains;
private final Map<String, SolrInfoMBean> infoRegistry;
private final IndexDeletionPolicyWrapper solrDelPolicy;
private final SolrSnapshotMetaDataManager snapshotMgr;
private final DirectoryFactory directoryFactory;
private IndexReaderFactory indexReaderFactory;
private final Codec codec;
@ -414,7 +416,19 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
} else {
delPolicy = new SolrDeletionPolicy();
}
return new IndexDeletionPolicyWrapper(delPolicy);
return new IndexDeletionPolicyWrapper(delPolicy, snapshotMgr);
}
private SolrSnapshotMetaDataManager initSnapshotMetaDataManager() {
try {
String dirName = getDataDir() + SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR + "/";
Directory snapshotDir = directoryFactory.get(dirName, DirContext.DEFAULT,
getSolrConfig().indexConfig.lockType);
return new SolrSnapshotMetaDataManager(this, snapshotDir);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
private void initListeners() {
@ -739,6 +753,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
initListeners();
this.snapshotMgr = initSnapshotMetaDataManager();
this.solrDelPolicy = initDeletionPolicy(delPolicy);
this.codec = initCodec(solrConfig, this.schema);
@ -1242,6 +1257,17 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
}
}
// Close the snapshots meta-data directory.
Directory snapshotsDir = snapshotMgr.getSnapshotsDir();
try {
this.directoryFactory.release(snapshotsDir);
} catch (Throwable e) {
SolrException.log(log,e);
if (e instanceof Error) {
throw (Error) e;
}
}
if (coreStateClosed) {
try {
@ -2343,6 +2369,14 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
return solrDelPolicy;
}
/**
* @return A reference of {@linkplain SolrSnapshotMetaDataManager}
* managing the persistent snapshots for this Solr core.
*/
public SolrSnapshotMetaDataManager getSnapshotMetaDataManager() {
return snapshotMgr;
}
public ReentrantLock getRuleExpiryLock() {
return ruleExpiryLock;
}

134
solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java Normal file
View File

@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class provides functionality required to handle the data files corresponding to Solr snapshots.
*/
public class SolrSnapshotManager {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
* This method deletes index files of the {@linkplain IndexCommit} for the specified generation number.
*
* @param dir The index directory storing the snapshot.
* @param gen The generation number for the {@linkplain IndexCommit}
* @throws IOException in case of I/O errors.
*/
public static void deleteIndexFiles ( Directory dir, Collection<SnapshotMetaData> snapshots, long gen ) throws IOException {
List<IndexCommit> commits = DirectoryReader.listCommits(dir);
Map<String, Integer> refCounts = buildRefCounts(snapshots, commits);
for (IndexCommit ic : commits) {
if (ic.getGeneration() == gen) {
deleteIndexFiles(dir,refCounts, ic);
break;
}
}
}
/**
* This method deletes all files not corresponding to a configured snapshot in the specified index directory.
*
* @param dir The index directory to search for.
* @throws IOException in case of I/O errors.
*/
public static void deleteNonSnapshotIndexFiles (Directory dir, Collection<SnapshotMetaData> snapshots) throws IOException {
List<IndexCommit> commits = DirectoryReader.listCommits(dir);
Map<String, Integer> refCounts = buildRefCounts(snapshots, commits);
Set<Long> snapshotGenNumbers = snapshots.stream()
.map(SnapshotMetaData::getGenerationNumber)
.collect(Collectors.toSet());
for (IndexCommit ic : commits) {
if (!snapshotGenNumbers.contains(ic.getGeneration())) {
deleteIndexFiles(dir,refCounts, ic);
}
}
}
/**
* This method computes reference count for the index files by taking into consideration
* (a) configured snapshots and (b) files sharing between two or more {@linkplain IndexCommit} instances.
*
* @param snapshots A collection of user configured snapshots
* @param commits A list of {@linkplain IndexCommit} instances
* @return A map containing reference count for each index file referred in one of the {@linkplain IndexCommit} instances.
* @throws IOException in case of I/O error.
*/
@VisibleForTesting
static Map<String, Integer> buildRefCounts (Collection<SnapshotMetaData> snapshots, List<IndexCommit> commits) throws IOException {
Map<String, Integer> result = new HashMap<>();
Map<Long, IndexCommit> commitsByGen = commits.stream().collect(
Collectors.toMap(IndexCommit::getGeneration, Function.identity()));
for(SnapshotMetaData md : snapshots) {
IndexCommit ic = commitsByGen.get(md.getGenerationNumber());
if (ic != null) {
Collection<String> fileNames = ic.getFileNames();
for(String fileName : fileNames) {
int refCount = result.getOrDefault(fileName, 0);
result.put(fileName, refCount+1);
}
}
}
return result;
}
/**
* This method deletes the index files associated with specified <code>indexCommit</code> provided they
* are not referred by some other {@linkplain IndexCommit}.
*
* @param dir The index directory containing the {@linkplain IndexCommit} to be deleted.
* @param refCounts A map containing reference counts for each file associated with every {@linkplain IndexCommit}
* in the specified directory.
* @param indexCommit The {@linkplain IndexCommit} whose files need to be deleted.
* @throws IOException in case of I/O errors.
*/
private static void deleteIndexFiles ( Directory dir, Map<String, Integer> refCounts, IndexCommit indexCommit ) throws IOException {
log.info("Deleting index files for index commit with generation {} in directory {}", indexCommit.getGeneration(), dir);
for (String fileName : indexCommit.getFileNames()) {
try {
// Ensure that a file being deleted is not referred by some other commit.
int ref = refCounts.getOrDefault(fileName, 0);
log.debug("Reference count for file {} is {}", fileName, ref);
if (ref == 0) {
dir.deleteFile(fileName);
}
} catch (IOException e) {
log.warn("Unable to delete file {} in directory {} due to exception {}", fileName, dir, e.getMessage());
}
}
}
}

416
solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java Normal file
View File

@ -0,0 +1,416 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class is responsible to manage the persistent snapshots meta-data for the Solr indexes. The
* persistent snapshots are implemented by relying on Lucene {@linkplain IndexDeletionPolicy}
* abstraction to configure a specific {@linkplain IndexCommit} to be retained. The
* {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to create/delete the Solr index
* snapshots.
*/
public class SolrSnapshotMetaDataManager {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata";
/**
* A class defining the meta-data for a specific snapshot.
*/
public static class SnapshotMetaData {
private String name;
private String indexDirPath;
private long generationNumber;
public SnapshotMetaData(String name, String indexDirPath, long generationNumber) {
super();
this.name = name;
this.indexDirPath = indexDirPath;
this.generationNumber = generationNumber;
}
public String getName() {
return name;
}
public String getIndexDirPath() {
return indexDirPath;
}
public long getGenerationNumber() {
return generationNumber;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("SnapshotMetaData[name=");
builder.append(name);
builder.append(", indexDirPath=");
builder.append(indexDirPath);
builder.append(", generation=");
builder.append(generationNumber);
builder.append("]");
return builder.toString();
}
}
/** Prefix used for the save file. */
public static final String SNAPSHOTS_PREFIX = "snapshots_";
private static final int VERSION_START = 0;
private static final int VERSION_CURRENT = VERSION_START;
private static final String CODEC_NAME = "solr-snapshots";
// The index writer which maintains the snapshots metadata
private long nextWriteGen;
private final Directory dir;
/** Used to map snapshot name to snapshot meta-data. */
protected final Map<String,SnapshotMetaData> nameToDetailsMapping = new LinkedHashMap<>();
/** Used to figure out the *current* index data directory path */
private final SolrCore solrCore;
/**
* A constructor.
*
* @param dir The directory where the snapshot meta-data should be stored. Enables updating
* the existing meta-data.
* @throws IOException in case of errors.
*/
public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir) throws IOException {
this(solrCore, dir, OpenMode.CREATE_OR_APPEND);
}
/**
* A constructor.
*
* @param dir The directory where the snapshot meta-data is stored.
* @param mode CREATE If previous meta-data should be erased.
* APPEND If previous meta-data should be read and updated.
* CREATE_OR_APPEND Creates a new meta-data structure if one does not exist
* Updates the existing structure if one exists.
* @throws IOException in case of errors.
*/
public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir, OpenMode mode) throws IOException {
this.solrCore = solrCore;
this.dir = dir;
if (mode == OpenMode.CREATE) {
deleteSnapshotMetadataFiles();
}
loadFromSnapshotMetadataFile();
if (mode == OpenMode.APPEND && nextWriteGen == 0) {
throw new IllegalStateException("no snapshots stored in this directory");
}
}
/**
* @return The snapshot meta-data directory
*/
public Directory getSnapshotsDir() {
return dir;
}
/**
* This method creates a new snapshot meta-data entry.
*
* @param name The name of the snapshot.
* @param indexDirPath The directory path where the index files are stored.
* @param gen The generation number for the {@linkplain IndexCommit} being snapshotted.
* @throws IOException in case of I/O errors.
*/
public synchronized void snapshot(String name, String indexDirPath, long gen) throws IOException {
Preconditions.checkNotNull(name);
log.info("Creating the snapshot named {} for core {} associated with index commit with generation {} in directory {}"
, name, solrCore.getName(), gen, indexDirPath);
if(nameToDetailsMapping.containsKey(name)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name " + name + " already exists");
}
SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen);
nameToDetailsMapping.put(name, d);
boolean success = false;
try {
persist();
success = true;
} finally {
if (!success) {
try {
release(name);
} catch (Exception e) {
// Suppress so we keep throwing original exception
}
}
}
}
/**
* This method deletes a previously created snapshot (if any).
*
* @param name The name of the snapshot to be deleted.
* @return The snapshot meta-data if the snapshot with the snapshot name exists.
* @throws IOException in case of I/O error
*/
public synchronized Optional<SnapshotMetaData> release(String name) throws IOException {
log.info("Deleting the snapshot named {} for core {}", name, solrCore.getName());
SnapshotMetaData result = nameToDetailsMapping.remove(Preconditions.checkNotNull(name));
if(result != null) {
boolean success = false;
try {
persist();
success = true;
} finally {
if (!success) {
nameToDetailsMapping.put(name, result);
}
}
}
return Optional.ofNullable(result);
}
/**
* This method returns if snapshot is created for the specified generation number in
* the *current* index directory.
*
* @param genNumber The generation number for the {@linkplain IndexCommit} to be checked.
* @return true if the snapshot is created.
* false otherwise.
*/
public synchronized boolean isSnapshotted(long genNumber) {
return !nameToDetailsMapping.isEmpty() && isSnapshotted(solrCore.getIndexDir(), genNumber);
}
/**
* This method returns if snapshot is created for the specified generation number in
* the specified index directory.
*
* @param genNumber The generation number for the {@linkplain IndexCommit} to be checked.
* @return true if the snapshot is created.
* false otherwise.
*/
public synchronized boolean isSnapshotted(String indexDirPath, long genNumber) {
return !nameToDetailsMapping.isEmpty()
&& nameToDetailsMapping.values().stream()
.anyMatch(entry -> entry.getIndexDirPath().equals(indexDirPath) && entry.getGenerationNumber() == genNumber);
}
/**
* This method returns the snapshot meta-data for the specified name (if it exists).
*
* @param name The name of the snapshot
* @return The snapshot meta-data if exists.
*/
public synchronized Optional<SnapshotMetaData> getSnapshotMetaData(String name) {
return Optional.ofNullable(nameToDetailsMapping.get(name));
}
/**
* @return A list of snapshots created so far.
*/
public synchronized List<String> listSnapshots() {
// We create a copy for thread safety.
return new ArrayList<>(nameToDetailsMapping.keySet());
}
/**
* This method returns a list of snapshots created in a specified index directory.
*
* @param indexDirPath The index directory path.
* @return a list snapshots stored in the specified directory.
*/
public synchronized Collection<SnapshotMetaData> listSnapshotsInIndexDir(String indexDirPath) {
return nameToDetailsMapping.values().stream()
.filter(entry -> indexDirPath.equals(entry.getIndexDirPath()))
.collect(Collectors.toList());
}
/**
* This method returns the {@linkplain IndexCommit} associated with the specified
* <code>commitName</code>. A snapshot with specified <code>commitName</code> must
* be created before invoking this method.
*
* @param commitName The name of persisted commit
* @return the {@linkplain IndexCommit}
* @throws IOException in case of I/O error.
*/
public Optional<IndexCommit> getIndexCommitByName(String commitName) throws IOException {
Optional<IndexCommit> result = Optional.empty();
Optional<SnapshotMetaData> metaData = getSnapshotMetaData(commitName);
if (metaData.isPresent()) {
String indexDirPath = metaData.get().getIndexDirPath();
long gen = metaData.get().getGenerationNumber();
Directory d = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE);
try {
result = DirectoryReader.listCommits(d)
.stream()
.filter(ic -> ic.getGeneration() == gen)
.findAny();
if (!result.isPresent()) {
log.warn("Unable to find commit with generation {} in the directory {}", gen, indexDirPath);
}
} finally {
solrCore.getDirectoryFactory().release(d);
}
} else {
log.warn("Commit with name {} is not persisted for core {}", commitName, solrCore.getName());
}
return result;
}
private synchronized void persist() throws IOException {
String fileName = SNAPSHOTS_PREFIX + nextWriteGen;
IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT);
boolean success = false;
try {
CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
out.writeVInt(nameToDetailsMapping.size());
for(Entry<String,SnapshotMetaData> ent : nameToDetailsMapping.entrySet()) {
out.writeString(ent.getKey());
out.writeString(ent.getValue().getIndexDirPath());
out.writeVLong(ent.getValue().getGenerationNumber());
}
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
} else {
IOUtils.close(out);
}
}
dir.sync(Collections.singletonList(fileName));
if (nextWriteGen > 0) {
String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
// exception OK: likely it didn't exist
IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
}
nextWriteGen++;
}
private synchronized void deleteSnapshotMetadataFiles() throws IOException {
for(String file : dir.listAll()) {
if (file.startsWith(SNAPSHOTS_PREFIX)) {
dir.deleteFile(file);
}
}
}
/**
* Reads the snapshot meta-data information from the given {@link Directory}.
*/
private synchronized void loadFromSnapshotMetadataFile() throws IOException {
log.info("Loading from snapshot metadata file...");
long genLoaded = -1;
IOException ioe = null;
List<String> snapshotFiles = new ArrayList<>();
for(String file : dir.listAll()) {
if (file.startsWith(SNAPSHOTS_PREFIX)) {
long gen = Long.parseLong(file.substring(SNAPSHOTS_PREFIX.length()));
if (genLoaded == -1 || gen > genLoaded) {
snapshotFiles.add(file);
Map<String, SnapshotMetaData> snapshotMetaDataMapping = new HashMap<>();
IndexInput in = dir.openInput(file, IOContext.DEFAULT);
try {
CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
int count = in.readVInt();
for(int i=0;i<count;i++) {
String name = in.readString();
String indexDirPath = in.readString();
long commitGen = in.readVLong();
snapshotMetaDataMapping.put(name, new SnapshotMetaData(name, indexDirPath, commitGen));
}
} catch (IOException ioe2) {
// Save first exception & throw in the end
if (ioe == null) {
ioe = ioe2;
}
} finally {
in.close();
}
genLoaded = gen;
nameToDetailsMapping.clear();
nameToDetailsMapping.putAll(snapshotMetaDataMapping);
}
}
}
if (genLoaded == -1) {
// Nothing was loaded...
if (ioe != null) {
// ... not for lack of trying:
throw ioe;
}
} else {
if (snapshotFiles.size() > 1) {
// Remove any broken / old snapshot files:
String curFileName = SNAPSHOTS_PREFIX + genLoaded;
for(String file : snapshotFiles) {
if (!curFileName.equals(file)) {
IOUtils.deleteFilesIgnoringExceptions(dir, file);
}
}
}
nextWriteGen = 1+genLoaded;
}
}
}

22
solr/core/src/java/org/apache/solr/core/snapshots/package-info.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Core classes for Solr's persistent snapshots functionality
*/
package org.apache.solr.core.snapshots;

16
solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
View File

@ -82,6 +82,9 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@ -468,9 +471,18 @@ public class IndexFetcher {
// let the system know we are changing dir's and the old one
// may be closed
if (indexDir != null) {
LOG.info("removing old index directory " + indexDir);
solrCore.getDirectoryFactory().doneWithDirectory(indexDir);
solrCore.getDirectoryFactory().remove(indexDir);
SolrSnapshotMetaDataManager snapshotsMgr = solrCore.getSnapshotMetaDataManager();
Collection<SnapshotMetaData> snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath);
// Delete the old index directory only if no snapshot exists in that directory.
if(snapshots.isEmpty()) {
LOG.info("removing old index directory " + indexDir);
solrCore.getDirectoryFactory().remove(indexDir);
} else {
SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots);
}
}
}

24
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -87,6 +87,7 @@ import org.apache.solr.core.SolrDeletionPolicy;
import org.apache.solr.core.SolrEventListener;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
@ -512,11 +513,24 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
numberToKeep = Integer.MAX_VALUE;
}
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
IndexCommit indexCommit = delPolicy.getLatestCommit();
IndexCommit indexCommit = null;
String commitName = params.get(CoreAdminParams.COMMIT_NAME);
if (commitName != null) {
SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager();
Optional<IndexCommit> commit = snapshotMgr.getIndexCommitByName(commitName);
if(commit.isPresent()) {
indexCommit = commit.get();
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName +
" for core " + core.getName());
}
} else {
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
indexCommit = delPolicy.getLatestCommit();
if (indexCommit == null) {
indexCommit = req.getSearcher().getIndexReader().getIndexCommit();
if (indexCommit == null) {
indexCommit = req.getSearcher().getIndexReader().getIndexCommit();
}
}
String location = params.get(CoreAdminParams.BACKUP_LOCATION);
@ -539,7 +553,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
// small race here before the commit point is saved
SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME));
SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME), commitName);
snapShooter.validateCreateSnapshot();
snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl);

18
solr/core/src/java/org/apache/solr/handler/RestoreCore.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.handler;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.Locale;
import java.util.concurrent.Callable;
@ -32,6 +33,9 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,6 +67,7 @@ public class RestoreCore implements Callable<Boolean> {
String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;
String indexDirPath = core.getIndexDir();
Directory restoreIndexDir = null;
Directory indexDir = null;
try {
@ -71,7 +76,7 @@ public class RestoreCore implements Callable<Boolean> {
DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
//Prefer local copy.
indexDir = core.getDirectoryFactory().get(core.getIndexDir(),
indexDir = core.getDirectoryFactory().get(indexDirPath,
DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
//Move all files from backupDir to restoreIndexDir
@ -130,7 +135,16 @@ public class RestoreCore implements Callable<Boolean> {
}
if (success) {
core.getDirectoryFactory().doneWithDirectory(indexDir);
core.getDirectoryFactory().remove(indexDir);
SolrSnapshotMetaDataManager snapshotsMgr = core.getSnapshotMetaDataManager();
Collection<SnapshotMetaData> snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath);
// Delete the old index directory only if no snapshot exists in that directory.
if (snapshots.isEmpty()) {
core.getDirectoryFactory().remove(indexDir);
} else {
SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots);
}
}
return true;

39
solr/core/src/java/org/apache/solr/handler/SnapShooter.java
View File

@ -26,12 +26,14 @@ import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.function.Consumer;
import com.google.common.base.Preconditions;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
@ -39,6 +41,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
@ -59,6 +62,7 @@ public class SnapShooter {
private URI baseSnapDirPath = null;
private URI snapshotDirPath = null;
private BackupRepository backupRepo = null;
private String commitName; // can be null
@Deprecated
public SnapShooter(SolrCore core, String location, String snapshotName) {
@ -71,14 +75,14 @@ public class SnapShooter {
} else {
snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
}
initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName);
initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName, null);
}
public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) {
initialize(backupRepo, core, location, snapshotName);
public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) {
initialize(backupRepo, core, location, snapshotName, commitName);
}
private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) {
private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) {
this.solrCore = Preconditions.checkNotNull(core);
this.backupRepo = Preconditions.checkNotNull(backupRepo);
this.baseSnapDirPath = backupRepo.createURI(Preconditions.checkNotNull(location)).normalize();
@ -90,6 +94,7 @@ public class SnapShooter {
directoryName = "snapshot." + fmt.format(new Date());
}
this.snapshotDirPath = backupRepo.createURI(location, directoryName);
this.commitName = commitName;
}
public BackupRepository getBackupRepository() {
@ -145,16 +150,26 @@ public class SnapShooter {
}
public NamedList createSnapshot() throws Exception {
IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy();
RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
try {
//TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first?
IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit();
deletionPolicy.saveCommitPoint(indexCommit.getGeneration());
try {
return createSnapshot(indexCommit);
} finally {
deletionPolicy.releaseCommitPoint(indexCommit.getGeneration());
if (commitName != null) {
SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
Optional<IndexCommit> commit = snapshotMgr.getIndexCommitByName(commitName);
if(commit.isPresent()) {
return createSnapshot(commit.get());
}
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to find an index commit with name " + commitName +
" for core " + solrCore.getName());
} else {
//TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first?
IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy();
IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit();
deletionPolicy.saveCommitPoint(indexCommit.getGeneration());
try {
return createSnapshot(indexCommit);
} finally {
deletionPolicy.releaseCommitPoint(indexCommit.getGeneration());
}
}
} finally {
searcher.decref();

125
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
View File

@ -34,6 +34,7 @@ import java.util.concurrent.Future;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@ -59,9 +60,13 @@ import org.apache.solr.core.CachingDirectoryFactory;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.RestoreCore;
import org.apache.solr.handler.SnapShooter;
import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp;
@ -794,22 +799,26 @@ enum CoreAdminOperation implements CoreAdminOp {
+ " parameter or as a default repository property");
}
try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
SnapShooter snapShooter = new SnapShooter(repository, core, location, name);
// validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
// But we want to throw. One reason is that
// this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
// file system. Otherwise, perhaps the FS location isn't shared -- we want an error.
if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Directory to contain snapshots doesn't exist: " + snapShooter.getLocation());
// An optional parameter to describe the snapshot to be backed-up. If this
// parameter is not supplied, the latest index commit is backed-up.
String commitName = params.get(CoreAdminParams.COMMIT_NAME);
try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
SnapShooter snapShooter = new SnapShooter(repository, core, location, name, commitName);
// validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
// But we want to throw. One reason is that
// this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
// file system. Otherwise, perhaps the FS location isn't shared -- we want an error.
if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Directory to contain snapshots doesn't exist: " + snapShooter.getLocation());
}
snapShooter.validateCreateSnapshot();
snapShooter.createSnapshot();
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Failed to backup core=" + cname + " because " + e, e);
}
snapShooter.validateCreateSnapshot();
snapShooter.createSnapshot();
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Failed to backup core=" + cname + " because " + e, e);
}
}),
RESTORECORE_OP(RESTORECORE, it -> {
@ -845,6 +854,92 @@ enum CoreAdminOperation implements CoreAdminOp {
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName());
}
}
}),
CREATESNAPSHOT_OP(CREATESNAPSHOT, it -> {
CoreContainer cc = it.handler.getCoreContainer();
final SolrParams params = it.req.getParams();
String commitName = params.required().get(CoreAdminParams.COMMIT_NAME);
String cname = params.required().get(CoreAdminParams.CORE);
try (SolrCore core = cc.getCore(cname)) {
if (core == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname);
}
String indexDirPath = core.getIndexDir();
IndexCommit ic = core.getDeletionPolicy().getLatestCommit();
if (ic == null) {
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
try {
ic = searcher.get().getIndexReader().getIndexCommit();
} finally {
searcher.decref();
}
}
SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager();
mgr.snapshot(commitName, indexDirPath, ic.getGeneration());
it.rsp.add("core", core.getName());
it.rsp.add("commitName", commitName);
it.rsp.add("indexDirPath", indexDirPath);
it.rsp.add("generation", ic.getGeneration());
}
}),
DELETESNAPSHOT_OP(DELETESNAPSHOT, it -> {
CoreContainer cc = it.handler.getCoreContainer();
final SolrParams params = it.req.getParams();
String commitName = params.required().get(CoreAdminParams.COMMIT_NAME);
String cname = params.required().get(CoreAdminParams.CORE);
try (SolrCore core = cc.getCore(cname)) {
if (core == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname);
}
SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager();
Optional<SnapshotMetaData> metadata = mgr.release(commitName);
if (metadata.isPresent()) {
long gen = metadata.get().getGenerationNumber();
String indexDirPath = metadata.get().getIndexDirPath();
// If the directory storing the snapshot is not the same as the *current* core
// index directory, then delete the files corresponding to this snapshot.
// Otherwise we leave the index files related to snapshot as is (assuming the
// underlying Solr IndexDeletionPolicy will clean them up appropriately).
if (!indexDirPath.equals(core.getIndexDir())) {
Directory d = core.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE);
try {
SolrSnapshotManager.deleteIndexFiles(d, mgr.listSnapshotsInIndexDir(indexDirPath), gen);
} finally {
core.getDirectoryFactory().release(d);
}
}
}
}
}),
LISTSNAPSHOTS_OP(LISTSNAPSHOTS, it -> {
CoreContainer cc = it.handler.getCoreContainer();
final SolrParams params = it.req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
try ( SolrCore core = cc.getCore(cname) ) {
if (core == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname);
}
SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager();
NamedList result = new NamedList();
for (String name : mgr.listSnapshots()) {
Optional<SnapshotMetaData> metadata = mgr.getSnapshotMetaData(name);
if ( metadata.isPresent() ) {
NamedList<String> props = new NamedList<>();
props.add("generation", String.valueOf(metadata.get().getGenerationNumber()));
props.add("indexDirPath", metadata.get().getIndexDirPath());
result.add(name, props);
}
}
it.rsp.add("snapshots", result);
}
});
final CoreAdminParams.CoreAdminAction action;

419
solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java Normal file
View File

@ -0,0 +1,419 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core.snapshots;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot;
import org.apache.solr.client.solrj.request.CoreAdminRequest.DeleteSnapshot;
import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.BackupRestoreUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
@Slow
public class TestSolrCoreSnapshots extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static long docsSeed; // see indexDocs()
@BeforeClass
public static void setupClass() throws Exception {
useFactory("solr.StandardDirectoryFactory");
configureCluster(1)// nodes
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
docsSeed = random().nextLong();
}
@AfterClass
public static void teardownClass() throws Exception {
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
}
@Test
public void testBackupRestore() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String collectionName = "SolrCoreSnapshots";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
create.process(solrClient);
String location = createTempDir().toFile().getAbsolutePath();
int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(1, collectionState.getActiveSlices().size());
Slice shard = collectionState.getActiveSlices().iterator().next();
assertEquals(1, shard.getReplicas().size());
Replica replica = shard.getReplicas().iterator().next();
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String backupName = TestUtil.randomSimpleString(random(), 1, 5);
String commitName = TestUtil.randomSimpleString(random(), 1, 5);
String duplicateName = commitName.concat("_duplicate");
try (
SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) {
SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName);
// Create another snapshot referring to the same index commit to verify the
// reference counting implementation during snapshot deletion.
SnapshotMetaData duplicateCommit = createSnapshot(adminClient, coreName, duplicateName);
assertEquals (metaData.getIndexDirPath(), duplicateCommit.getIndexDirPath());
assertEquals (metaData.getGenerationNumber(), duplicateCommit.getGenerationNumber());
// Delete all documents
masterClient.deleteByQuery("*:*");
masterClient.commit();
BackupRestoreUtils.verifyDocs(0, cluster.getSolrClient(), collectionName);
// Verify that the index directory contains at least 2 index commits - one referred by the snapshots
// and the other containing document deletions.
{
List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
assertTrue(2 <= commits.size());
}
// Backup the earlier created snapshot.
{
Map<String,String> params = new HashMap<>();
params.put("name", backupName);
params.put("commitName", commitName);
params.put("location", location);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
// Restore the backup
{
Map<String,String> params = new HashMap<>();
params.put("name", "snapshot." + backupName);
params.put("location", location);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
BackupRestoreUtils.verifyDocs(nDocs, cluster.getSolrClient(), collectionName);
}
// Verify that the old index directory (before restore) contains only those index commits referred by snapshots.
{
List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
assertEquals(1, commits.size());
assertEquals(metaData.getGenerationNumber(), commits.get(0).getGeneration());
}
// Delete first snapshot
deleteSnapshot(adminClient, coreName, commitName);
// Verify that corresponding index files have NOT been deleted (due to reference counting).
assertFalse(listCommits(metaData.getIndexDirPath()).isEmpty());
// Delete second snapshot
deleteSnapshot(adminClient, coreName, duplicateCommit.getName());
// Verify that corresponding index files have been deleted.
assertTrue(listCommits(duplicateCommit.getIndexDirPath()).isEmpty());
}
}
@Test
public void testHandlingSharedIndexFiles() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String collectionName = "SolrCoreSnapshots_IndexFileSharing";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
create.process(solrClient);
int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(1, collectionState.getActiveSlices().size());
Slice shard = collectionState.getActiveSlices().iterator().next();
assertEquals(1, shard.getReplicas().size());
Replica replica = shard.getReplicas().iterator().next();
String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String backupName = TestUtil.randomSimpleString(random(), 1, 5);
String location = createTempDir().toFile().getAbsolutePath();
try (
SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) {
int numTests = TestUtil.nextInt(random(), 2, 5);
List<SnapshotMetaData> snapshots = new ArrayList<>(numTests);
// Create multiple commits and create a snapshot per commit.
// This should result in Lucene reusing some of the segments for later index commits.
for (int attempt=0; attempt<numTests; attempt++) {
if (nDocs > 0) {
//Delete a few docs
int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
for(int i=0; i<numDeletes; i++) {
masterClient.deleteByQuery("id:" + i);
}
}
// Add a few more
int moreAdds = TestUtil.nextInt(random(), 1, 100);
for (int i = 0; i < moreAdds; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i + nDocs);
doc.addField("name", "name = " + (i + nDocs));
masterClient.add(doc);
}
masterClient.commit();
// Create a snapshot
snapshots.add(createSnapshot(adminClient, coreName, "snapshot_" + attempt));
}
// Backup the earlier created snapshot.
{
Map<String,String> params = new HashMap<>();
params.put("name", backupName);
params.put("commitName", snapshots.get(0).getName());
params.put("location", location);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
// Restore the backup. The purpose of the restore operation is to change the *current* index directory.
// This is required since we delegate the file deletion to underlying IndexDeletionPolicy in case of
// *current* index directory. Hence for the purpose of this test, we want to ensure that the created
// snapshots are NOT in the *current* index directory.
{
Map<String,String> params = new HashMap<>();
params.put("name", "snapshot." + backupName);
params.put("location", location);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
}
{
SnapshotMetaData snapshotMetaData = snapshots.get(0);
List<IndexCommit> commits = listCommits(snapshotMetaData.getIndexDirPath());
// Check if number of index commits are > 0 to ensure index file sharing.
assertTrue(commits.size() > 0);
Map<String,Integer> refCounts = SolrSnapshotManager.buildRefCounts(snapshots, commits);
Optional<IndexCommit> ic = commits.stream()
.filter(entry -> entry.getGeneration() == snapshotMetaData.getGenerationNumber())
.findFirst();
assertTrue(ic.isPresent());
Collection<String> nonSharedFiles = new ArrayList<>();
Collection<String> sharedFiles = new ArrayList<>();
for (String fileName : ic.get().getFileNames()) {
if (refCounts.getOrDefault(fileName, 0) > 1) {
sharedFiles.add(fileName);
} else {
nonSharedFiles.add(fileName);
}
}
// Delete snapshot
deleteSnapshot(adminClient, coreName, snapshotMetaData.getName());
// Verify that the shared files are not deleted.
for (String fileName : sharedFiles) {
Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName);
assertTrue(path + " should exist.", Files.exists(path));
}
// Verify that the non-shared files are deleted.
for (String fileName : nonSharedFiles) {
Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName);
assertFalse(path + " should not exist.", Files.exists(path));
}
}
}
}
@Test
public void testIndexOptimization() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String collectionName = "SolrCoreSnapshots_IndexOptimization";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
create.process(solrClient);
int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(1, collectionState.getActiveSlices().size());
Slice shard = collectionState.getActiveSlices().iterator().next();
assertEquals(1, shard.getReplicas().size());
Replica replica = shard.getReplicas().iterator().next();
String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String commitName = TestUtil.randomSimpleString(random(), 1, 5);
try (
SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) {
SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName);
int numTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
for (int attempt=0; attempt<numTests; attempt++) {
//Modify existing index before we call optimize.
if (nDocs > 0) {
//Delete a few docs
int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
for(int i=0; i<numDeletes; i++) {
masterClient.deleteByQuery("id:" + i);
}
//Add a few more
int moreAdds = TestUtil.nextInt(random(), 1, 100);
for (int i=0; i<moreAdds; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i + nDocs);
doc.addField("name", "name = " + (i + nDocs));
masterClient.add(doc);
}
masterClient.commit();
}
}
// Before invoking optimize command, verify that the index directory contains multiple commits (including the one we snapshotted earlier).
{
Collection<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
// Verify that multiple index commits are stored in this directory.
assertTrue(commits.size() > 0);
// Verify that the snapshot commit is present in this directory.
assertTrue(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
}
// Optimize the index.
masterClient.optimize(true, true, 1);
// After invoking optimize command, verify that the index directory contains multiple commits (including the one we snapshotted earlier).
{
List<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
// Verify that multiple index commits are stored in this directory.
assertTrue(commits.size() > 1);
// Verify that the snapshot commit is present in this directory.
assertTrue(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
}
// Delete the snapshot
deleteSnapshot(adminClient, coreName, metaData.getName());
// Add few documents. Without this the optimize command below does not take effect.
{
int moreAdds = TestUtil.nextInt(random(), 1, 100);
for (int i=0; i<moreAdds; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i + nDocs);
doc.addField("name", "name = " + (i + nDocs));
masterClient.add(doc);
}
masterClient.commit();
}
// Optimize the index.
masterClient.optimize(true, true, 1);
// Verify that the index directory contains only 1 index commit (which is not the same as the snapshotted commit).
Collection<IndexCommit> commits = listCommits(metaData.getIndexDirPath());
assertTrue(commits.size() == 1);
assertFalse(commits.stream().filter(x -> x.getGeneration() == metaData.getGenerationNumber()).findFirst().isPresent());
}
}
private SnapshotMetaData createSnapshot (SolrClient adminClient, String coreName, String commitName) throws Exception {
CreateSnapshot req = new CreateSnapshot(commitName);
req.setCoreName(coreName);
adminClient.request(req);
Collection<SnapshotMetaData> snapshots = listSnapshots(adminClient, coreName);
Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
assertTrue(metaData.isPresent());
return metaData.get();
}
private void deleteSnapshot(SolrClient adminClient, String coreName, String commitName) throws Exception {
DeleteSnapshot req = new DeleteSnapshot(commitName);
req.setCoreName(coreName);
adminClient.request(req);
Collection<SnapshotMetaData> snapshots = listSnapshots(adminClient, coreName);
assertFalse(snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst().isPresent());
}
private Collection<SnapshotMetaData> listSnapshots(SolrClient adminClient, String coreName) throws Exception {
ListSnapshots req = new ListSnapshots();
req.setCoreName(coreName);
NamedList resp = adminClient.request(req);
assertTrue( resp.get("snapshots") instanceof NamedList );
NamedList apiResult = (NamedList) resp.get("snapshots");
List<SnapshotMetaData> result = new ArrayList<>(apiResult.size());
for(int i = 0 ; i < apiResult.size(); i++) {
String commitName = apiResult.getName(i);
String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get("indexDirPath");
long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get("generation"));
result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber));
}
return result;
}
private List<IndexCommit> listCommits(String directory) throws Exception {
SimpleFSDirectory dir = new SimpleFSDirectory(Paths.get(directory));
try {
return DirectoryReader.listCommits(dir);
} catch (IndexNotFoundException ex) {
// This can happen when the delete snapshot functionality cleans up the index files (when the directory
// storing these files is not the *current* index directory).
return Collections.emptyList();
}
}
}

37
solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java
View File

@ -18,11 +18,15 @@
package org.apache.solr.handler;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
@ -64,4 +68,37 @@ public class BackupRestoreUtils extends LuceneTestCase {
assertEquals(0, response.getStatus());
assertEquals(nDocs, response.getResults().getNumFound());
}
public static void runCoreAdminCommand(String baseUrl, String coreName, String action, Map<String,String> params) throws IOException {
StringBuilder builder = new StringBuilder();
builder.append(baseUrl);
builder.append("/admin/cores?action=");
builder.append(action);
builder.append("&core=");
builder.append(coreName);
for (Map.Entry<String,String> p : params.entrySet()) {
builder.append("&");
builder.append(p.getKey());
builder.append("=");
builder.append(p.getValue());
}
String masterUrl = builder.toString();
executeHttpRequest(masterUrl);
}
public static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException {
String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName;
executeHttpRequest(masterUrl);
}
static void executeHttpRequest(String requestUrl) throws IOException {
InputStream stream = null;
try {
URL url = new URL(requestUrl);
stream = url.openStream();
stream.close();
} finally {
IOUtils.closeQuietly(stream);
}
}
}

46
solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
View File

@ -18,11 +18,11 @@
package org.apache.solr.handler;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.commons.io.IOUtils;
@ -44,6 +44,7 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
@ -176,16 +177,19 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) {
// Create a backup.
if (testViaReplicationHandler) {
log.info("Running Backup/restore via replication handler");
runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
log.info("Running Backup via replication handler");
BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null);
while (!checkBackupStatus.success) {
checkBackupStatus.fetchStatus();
Thread.sleep(1000);
}
} else {
log.info("Running Backup/restore via core admin api");
runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), "hdfs", backupName);
log.info("Running Backup via core admin api");
Map<String,String> params = new HashMap<>();
params.put("name", backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
@ -214,38 +218,22 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
}
// Snapshooter prefixes "snapshot." to the backup name.
if (testViaReplicationHandler) {
log.info("Running Restore via replication handler");
// Snapshooter prefixes "snapshot." to the backup name.
runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
while (!TestRestoreCore.fetchRestoreStatus(baseUrl, coreName)) {
Thread.sleep(1000);
}
} else {
runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), "hdfs", "snapshot." + backupName);
log.info("Running Restore via core admin api");
Map<String,String> params = new HashMap<>();
params.put("name", "snapshot." + backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
}
//See if restore was successful by checking if all the docs are present again
BackupRestoreUtils.verifyDocs(nDocs, masterClient, coreName);
}
}
}
static void runCoreAdminCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException {
String masterUrl = baseUrl + "/admin/cores?action=" + action + "&core="+coreName+"&repository="+repoName+"&name="+backupName;
executeHttpRequest(masterUrl);
}
static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException {
String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName;
executeHttpRequest(masterUrl);
}
static void executeHttpRequest(String requestUrl) throws IOException {
InputStream stream = null;
try {
URL url = new URL(requestUrl);
stream = url.openStream();
stream.close();
} finally {
IOUtils.closeQuietly(stream);
}
}
}

20
solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
View File

@ -20,6 +20,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -66,6 +67,7 @@ import org.apache.solr.core.CachingDirectoryFactory;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.StandardDirectoryFactory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.util.FileUtils;
import org.junit.After;
import org.junit.Before;
@ -896,8 +898,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
CachingDirectoryFactory dirFactory = (CachingDirectoryFactory) core.getDirectoryFactory();
synchronized (dirFactory) {
Set<String> livePaths = dirFactory.getLivePaths();
// one for data, one for hte index under data
assertEquals(livePaths.toString(), 2, livePaths.size());
// one for data, one for hte index under data and one for the snapshot metadata.
assertEquals(livePaths.toString(), 3, livePaths.size());
// :TODO: assert that one of the paths is a subpath of hte other
}
if (dirFactory instanceof StandardDirectoryFactory) {
@ -908,14 +910,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
private int indexDirCount(String ddir) {
String[] list = new File(ddir).list();
int cnt = 0;
for (String file : list) {
if (!file.endsWith(".properties")) {
cnt++;
String[] list = new File(ddir).list(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
File f = new File(dir, name);
return f.isDirectory() && !SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR.equals(name);
}
}
return cnt;
});
return list.length;
}
private void pullFromMasterToSlave() throws MalformedURLException,

57
solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
View File

@ -453,6 +453,63 @@ public class CoreAdminRequest extends SolrRequest<CoreAdminResponse> {
}
public static class CreateSnapshot extends CoreAdminRequest {
private String commitName;
public CreateSnapshot(String commitName) {
super();
this.action = CoreAdminAction.CREATESNAPSHOT;
if(commitName == null) {
throw new NullPointerException("Please specify non null value for commitName parameter.");
}
this.commitName = commitName;
}
public String getCommitName() {
return commitName;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
params.set(CoreAdminParams.COMMIT_NAME, this.commitName);
return params;
}
}
public static class DeleteSnapshot extends CoreAdminRequest {
private String commitName;
public DeleteSnapshot(String commitName) {
super();
this.action = CoreAdminAction.DELETESNAPSHOT;
if(commitName == null) {
throw new NullPointerException("Please specify non null value for commitName parameter.");
}
this.commitName = commitName;
}
public String getCommitName() {
return commitName;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
params.set(CoreAdminParams.COMMIT_NAME, this.commitName);
return params;
}
}
public static class ListSnapshots extends CoreAdminRequest {
public ListSnapshots() {
super();
this.action = CoreAdminAction.LISTSNAPSHOTS;
}
}
public CoreAdminRequest()
{
super( METHOD.GET, "/admin/cores" );

10
solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
View File

@ -118,6 +118,11 @@ public abstract class CoreAdminParams
*/
public static final String BACKUP_LOCATION = "location";
/**
* A parameter to specify the name of the commit to be stored during the backup operation.
*/
public static final String COMMIT_NAME = "commitName";
public enum CoreAdminAction {
STATUS(true),
UNLOAD,
@ -141,7 +146,10 @@ public abstract class CoreAdminParams
INVOKE,
//Internal APIs to backup and restore a core
BACKUPCORE,
RESTORECORE;
RESTORECORE,
CREATESNAPSHOT,
DELETESNAPSHOT,
LISTSNAPSHOTS;
public final boolean isRead;