From 2149a9403d3483e212f2eb900b57ba16d2cee3fe Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 2 Jun 2014 13:10:54 -0400 Subject: [PATCH] Improve deletion of corrupted snapshots Makes it possible to delete snapshots that are missing some of the metadata files. This can happen if snapshot creation failed because repository drive ran out of disk space. Closes #6383 --- .../blobstore/BlobStoreRepository.java | 112 ++++++++---------- .../SharedClusterSnapshotRestoreTests.java | 39 ++++++ 2 files changed, 91 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f5119aa0c92..b25fb796aa2 100644 --- a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -259,7 +259,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent indices) { - MetaData metaData; - try { - byte[] data = snapshotsBlobContainer.readBlobFully(metaDataBlobName(snapshotId)); - metaData = readMetaData(data); - } catch (FileNotFoundException | NoSuchFileException ex) { - throw new SnapshotMissingException(snapshotId, ex); - } catch (IOException ex) { - throw new SnapshotException(snapshotId, "failed to get snapshots", ex); - } - MetaData.Builder metaDataBuilder = MetaData.builder(metaData); - for (String index : indices) { - BlobPath indexPath = basePath().add("indices").add(index); - ImmutableBlobContainer indexMetaDataBlobContainer = blobStore().immutableBlobContainer(indexPath); - XContentParser parser = null; - try { - byte[] data = indexMetaDataBlobContainer.readBlobFully(snapshotBlobName(snapshotId)); - parser = XContentHelper.createParser(data, 0, data.length); - XContentParser.Token token; - if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) { - IndexMetaData indexMetaData = IndexMetaData.Builder.fromXContent(parser); - if ((token = parser.nextToken()) == XContentParser.Token.END_OBJECT) { - metaDataBuilder.put(indexMetaData, false); - continue; - } - } - throw new ElasticsearchParseException("unexpected token [" + token + "]"); - } catch (IOException ex) { - throw new SnapshotException(snapshotId, "failed to read metadata", ex); - } finally { - if (parser != null) { - parser.close(); - } - } - } - return metaDataBuilder.build(); + return readSnapshotMetaData(snapshotId, indices, false); } /** @@ -439,6 +407,48 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent indices, boolean ignoreIndexErrors) { + MetaData metaData; + try { + byte[] data = snapshotsBlobContainer.readBlobFully(metaDataBlobName(snapshotId)); + metaData = readMetaData(data); + } catch (FileNotFoundException | NoSuchFileException ex) { + throw new SnapshotMissingException(snapshotId, ex); + } catch (IOException ex) { + throw new SnapshotException(snapshotId, "failed to get snapshots", ex); + } + MetaData.Builder metaDataBuilder = MetaData.builder(metaData); + for (String index : indices) { + BlobPath indexPath = basePath().add("indices").add(index); + ImmutableBlobContainer indexMetaDataBlobContainer = blobStore().immutableBlobContainer(indexPath); + try { + byte[] data = indexMetaDataBlobContainer.readBlobFully(snapshotBlobName(snapshotId)); + try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) { + XContentParser.Token token; + if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) { + IndexMetaData indexMetaData = IndexMetaData.Builder.fromXContent(parser); + if ((token = parser.nextToken()) == XContentParser.Token.END_OBJECT) { + metaDataBuilder.put(indexMetaData, false); + continue; + } + } + if (!ignoreIndexErrors) { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } else { + logger.warn("[{}] [{}] unexpected token while reading snapshot metadata [{}]", snapshotId, index, token); + } + } + } catch (IOException ex) { + if (!ignoreIndexErrors) { + throw new SnapshotException(snapshotId, "failed to read metadata", ex); + } else { + logger.warn("[{}] [{}] failed to read metadata for index", snapshotId, index, ex); + } + } + } + return metaDataBuilder.build(); + } + /** * Configures RateLimiter based on repository and global settings * @@ -465,9 +475,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent readSnapshotList() throws IOException { byte[] data = snapshotsBlobContainer.readBlobFully(SNAPSHOTS_FILE); ArrayList snapshots = new ArrayList<>(); - XContentParser parser = null; - try { - parser = XContentHelper.createParser(data, 0, data.length); + try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) { if (parser.nextToken() == XContentParser.Token.START_OBJECT) { if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); @@ -630,10 +626,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent creating repository at " + repo.getAbsolutePath()); + assertAcked(client.admin().cluster().preparePutRepository("test-repo") + .setType("fs").setSettings(ImmutableSettings.settingsBuilder() + .put("location", repo) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000)))); + + createIndex("test-idx-1", "test-idx-2"); + ensureYellow(); + logger.info("--> indexing some data"); + indexRandom(true, + client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"), + client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar")); + + logger.info("--> creating snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + logger.info("--> delete index metadata and shard metadata"); + File indices = new File(repo, "indices"); + File testIndex1 = new File(indices, "test-idx-1"); + File testIndex2 = new File(indices, "test-idx-2"); + File testIndex2Shard0 = new File(testIndex2, "0"); + new File(testIndex1, "snapshot-test-snap-1").delete(); + new File(testIndex2Shard0, "snapshot-test-snap-1").delete(); + + logger.info("--> delete snapshot"); + client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); + + logger.info("--> make sure snapshot doesn't exist"); + assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class); + } + @Test @TestLogging("snapshots:TRACE") public void snapshotClosedIndexTest() throws Exception {