From 73faa2e5edb611d7f8dc87e330b21219400fa1c7 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Tue, 3 Feb 2015 20:27:45 -0500
Subject: [PATCH] Snapshot/Restore: Allow deletion of snapshots with corrupted
 snapshot files

Improve resiliency of snapshot deletion operation by allowing deletion of snapshot with corrupted snapshot files.

Closes #9534
---
 .../blobstore/BlobStoreRepository.java        | 13 +++--
 .../SharedClusterSnapshotRestoreTests.java    | 47 +++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 894550811fb..c76a03f9bb3 100644
--- a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -259,10 +259,17 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      */
     @Override
     public void deleteSnapshot(SnapshotId snapshotId) {
-        Snapshot snapshot = readSnapshot(snapshotId);
+        ImmutableList<String> indices = ImmutableList.of();
+        try {
+            indices = readSnapshot(snapshotId).indices();
+        } catch (SnapshotMissingException ex) {
+            throw ex;
+        } catch (SnapshotException | ElasticsearchParseException ex) {
+            logger.warn("cannot read snapshot file [{}]", ex, snapshotId);
+        }
         MetaData metaData = null;
         try {
-            metaData = readSnapshotMetaData(snapshotId, snapshot.indices(), true);
+            metaData = readSnapshotMetaData(snapshotId, indices, true);
         } catch (IOException | SnapshotException ex) {
             logger.warn("cannot read metadata for snapshot [{}]", ex, snapshotId);
         }
@@ -284,7 +291,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             }
             writeSnapshotList(snapshotIds);
             // Now delete all indices
-            for (String index : snapshot.indices()) {
+            for (String index : indices) {
                 BlobPath indexPath = basePath().add("indices").add(index);
                 BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
                 try {
diff --git a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
index c67da7f628e..86cc0181ee2 100644
--- a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
+++ b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
@@ -54,8 +54,13 @@ import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
 import org.junit.Test;
 
+import java.io.FileOutputStream;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -821,6 +826,48 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
         assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class);
     }
 
+    @Test
+    public void deleteSnapshotWithCorruptedSnapshotFileTest() throws Exception {
+        Client client = client();
+
+        Path repo = newTempDirPath();
+        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+                .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
+                        .put("location", repo)
+                        .put("compress", false)
+                        .put("chunk_size", randomIntBetween(100, 1000))));
+
+        createIndex("test-idx-1", "test-idx-2");
+        ensureYellow();
+        logger.info("--> indexing some data");
+        indexRandom(true,
+                client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+
+        logger.info("--> creating snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        logger.info("--> truncate snapshot file to make it unreadable");
+        Path snapshotPath = repo.resolve("snapshot-test-snap-1");
+        try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) {
+            outChan.truncate(randomInt(10));
+        }
+        logger.info("--> delete snapshot");
+        client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
+
+        logger.info("--> make sure snapshot doesn't exist");
+        assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class);
+
+        logger.info("--> make sure that we can create the snapshot again");
+        createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+    }
+
+
     @Test
     public void snapshotClosedIndexTest() throws Exception {
         Client client = client();