honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-06 04:58:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Snapshot/Restore: keep the last legacy checksums file at the end of restore

Browse Source 
This commit fixes the issue caused by restore process deleting all legacy checksum files at the end of restore process. Instead it keeps the latest version of the checksum intact. The issue manifests itself in losing checksum for all legacy files restored into post 1.3.0 cluster, which in turn causes unnecessary snapshotting of files that didn't change.

Fixes #8119
This commit is contained in:
Igor Motov 2014-11-05 22:11:10 -08:00
parent b0dde6ee4a
commit fd8a56dc0b
2 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
View File

@ -804,7 +804,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
/// now, go over and clean files that are in the store, but were not in the snapshot
try {
for (String storeFile : store.directory().listAll()) {
if (!snapshot.containPhysicalIndexFile(storeFile)) {
if (!Store.isChecksum(storeFile) && !snapshot.containPhysicalIndexFile(storeFile)) {
try {
store.directory().deleteFile(storeFile);
} catch (IOException e) {

38
src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Directories;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
@ -580,6 +581,8 @@ public class Store extends AbstractIndexShardComponent implements CloseableIndex
* @see StoreFileMetaData
*/
public final static class MetadataSnapshot implements Iterable<StoreFileMetaData> {
private static final ESLogger logger = Loggers.getLogger(MetadataSnapshot.class);
private final Map<String, StoreFileMetaData> metadata;
public static final MetadataSnapshot EMPTY = new MetadataSnapshot();
@ -643,6 +646,16 @@ public class Store extends AbstractIndexShardComponent implements CloseableIndex
return builder.build();
}
/**
* Reads legacy checksum files found in the directory.
*
* Files are expected to start with _checksums- prefix
* followed by long file version. Only file with the highest version is read, all other files are ignored.
*
* @param directory the directory to read checksums from
* @return a map of file checksums and the checksum file version
* @throws IOException
*/
static Tuple<Map<String, String>, Long> readLegacyChecksums(Directory directory) throws IOException {
synchronized (directory) {
long lastFound = -1;
@ -665,6 +678,30 @@ public class Store extends AbstractIndexShardComponent implements CloseableIndex
}
}
/**
* Deletes all checksum files with version lower than newVersion.
*
* @param directory the directory to clean
* @param newVersion the latest checksum file version
* @throws IOException
*/
static void cleanLegacyChecksums(Directory directory, long newVersion) throws IOException {
synchronized (directory) {
for (String name : directory.listAll()) {
if (isChecksum(name)) {
long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length()));
if (current < newVersion) {
try {
directory.deleteFile(name);
} catch (IOException ex) {
logger.debug("can't delete old checksum file [{}]", ex, name);
}
}
}
}
}
}
private static void checksumFromLuceneFile(Directory directory, String file, ImmutableMap.Builder<String, StoreFileMetaData> builder, ESLogger logger, Version version, boolean readFileAsHash) throws IOException {
final String checksum;
final BytesRefBuilder fileHash = new BytesRefBuilder();
@ -877,6 +914,7 @@ public class Store extends AbstractIndexShardComponent implements CloseableIndex
output.writeStringStringMap(checksums);
}
directory.sync(Collections.singleton(checksumName));
MetadataSnapshot.cleanLegacyChecksums(directory, nextVersion);
}
public void clear() {