diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 573bb0ea263..ad8541ce9fd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -88,7 +88,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction 0) { final Set requestedSnapshotNames = Sets.newHashSet(request.snapshots()); - final Map matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream() + final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName); + final Map matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream() .filter(s -> requestedSnapshotNames.contains(s.getName())) .collect(Collectors.toMap(SnapshotId::getName, Function.identity())); for (final String snapshotName : request.snapshots()) { @@ -220,6 +223,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction shardStatusBuilder = new ArrayList<>(); @@ -243,7 +248,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction> indexSnapshots; + /** + * The snapshots that are no longer compatible with the current cluster ES version. + */ + private final List incompatibleSnapshotIds; - private RepositoryData(long genId, List snapshotIds, Map> indexSnapshots) { + public RepositoryData(long genId, List snapshotIds, Map> indexSnapshots, + List incompatibleSnapshotIds) { this.genId = genId; this.snapshotIds = Collections.unmodifiableList(snapshotIds); this.indices = Collections.unmodifiableMap(indexSnapshots.keySet() .stream() .collect(Collectors.toMap(IndexId::getName, Function.identity()))); this.indexSnapshots = Collections.unmodifiableMap(indexSnapshots); - } - - /** - * Creates an instance of {@link RepositoryData} on a fresh repository (one that has no index-N files). - */ - public static RepositoryData initRepositoryData(List snapshotIds, Map> indexSnapshots) { - return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, indexSnapshots); + this.incompatibleSnapshotIds = Collections.unmodifiableList(incompatibleSnapshotIds); } protected RepositoryData copy() { - return new RepositoryData(genId, snapshotIds, indexSnapshots); + return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds); } /** @@ -104,6 +104,25 @@ public final class RepositoryData implements ToXContent { return snapshotIds; } + /** + * Returns an immutable collection of the snapshot ids in the repository that are incompatible with the + * current ES version. + */ + public List getIncompatibleSnapshotIds() { + return incompatibleSnapshotIds; + } + + /** + * Returns an immutable collection of all the snapshot ids in the repository, both active and + * incompatible snapshots. + */ + public List getAllSnapshotIds() { + List allSnapshotIds = new ArrayList<>(snapshotIds.size() + incompatibleSnapshotIds.size()); + allSnapshotIds.addAll(snapshotIds); + allSnapshotIds.addAll(incompatibleSnapshotIds); + return Collections.unmodifiableList(allSnapshotIds); + } + /** * Returns an unmodifiable map of the index names to {@link IndexId} in the repository. */ @@ -139,7 +158,7 @@ public final class RepositoryData implements ToXContent { allIndexSnapshots.put(indexId, ids); } } - return new RepositoryData(genId, snapshots, allIndexSnapshots); + return new RepositoryData(genId, snapshots, allIndexSnapshots, incompatibleSnapshotIds); } /** @@ -168,7 +187,21 @@ public final class RepositoryData implements ToXContent { indexSnapshots.put(indexId, set); } - return new RepositoryData(genId, newSnapshotIds, indexSnapshots); + return new RepositoryData(genId, newSnapshotIds, indexSnapshots, incompatibleSnapshotIds); + } + + /** + * Returns a new {@link RepositoryData} instance containing the same snapshot data as the + * invoking instance, with the given incompatible snapshots added to the new instance. + */ + public RepositoryData addIncompatibleSnapshots(final List incompatibleSnapshotIds) { + List newSnapshotIds = new ArrayList<>(this.snapshotIds); + List newIncompatibleSnapshotIds = new ArrayList<>(this.incompatibleSnapshotIds); + for (SnapshotId snapshotId : incompatibleSnapshotIds) { + newSnapshotIds.remove(snapshotId); + newIncompatibleSnapshotIds.add(snapshotId); + } + return new RepositoryData(this.genId, newSnapshotIds, this.indexSnapshots, newIncompatibleSnapshotIds); } /** @@ -182,6 +215,13 @@ public final class RepositoryData implements ToXContent { return snapshotIds; } + /** + * Initializes the indices in the repository metadata; returns a new instance. + */ + public RepositoryData initIndices(final Map> indexSnapshots) { + return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds); + } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -193,12 +233,13 @@ public final class RepositoryData implements ToXContent { @SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj; return snapshotIds.equals(that.snapshotIds) && indices.equals(that.indices) - && indexSnapshots.equals(that.indexSnapshots); + && indexSnapshots.equals(that.indexSnapshots) + && incompatibleSnapshotIds.equals(that.incompatibleSnapshotIds); } @Override public int hashCode() { - return Objects.hash(snapshotIds, indices, indexSnapshots); + return Objects.hash(snapshotIds, indices, indexSnapshots, incompatibleSnapshotIds); } /** @@ -247,11 +288,15 @@ public final class RepositoryData implements ToXContent { } private static final String SNAPSHOTS = "snapshots"; + private static final String INCOMPATIBLE_SNAPSHOTS = "incompatible-snapshots"; private static final String INDICES = "indices"; private static final String INDEX_ID = "id"; - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + /** + * Writes the snapshots metadata and the related indices metadata to x-content, omitting the + * incompatible snapshots. + */ + public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException { builder.startObject(); // write the snapshots list builder.startArray(SNAPSHOTS); @@ -278,7 +323,10 @@ public final class RepositoryData implements ToXContent { return builder; } - public static RepositoryData fromXContent(final XContentParser parser, final long genId) throws IOException { + /** + * Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata. + */ + public static RepositoryData snapshotsFromXContent(final XContentParser parser, long genId) throws IOException { List snapshots = new ArrayList<>(); Map> indexSnapshots = new HashMap<>(); if (parser.nextToken() == XContentParser.Token.START_OBJECT) { @@ -327,7 +375,51 @@ public final class RepositoryData implements ToXContent { } else { throw new ElasticsearchParseException("start object expected"); } - return new RepositoryData(genId, snapshots, indexSnapshots); + return new RepositoryData(genId, snapshots, indexSnapshots, Collections.emptyList()); + } + + /** + * Writes the incompatible snapshot ids to x-content. + */ + public XContentBuilder incompatibleSnapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) + throws IOException { + + builder.startObject(); + // write the incompatible snapshots list + builder.startArray(INCOMPATIBLE_SNAPSHOTS); + for (final SnapshotId snapshot : getIncompatibleSnapshotIds()) { + snapshot.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + /** + * Reads the incompatible snapshot ids from x-content, loading them into a new instance of {@link RepositoryData} + * that is created from the invoking instance, plus the incompatible snapshots that are read from x-content. + */ + public RepositoryData incompatibleSnapshotsFromXContent(final XContentParser parser) throws IOException { + List incompatibleSnapshotIds = new ArrayList<>(); + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + while (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if (INCOMPATIBLE_SNAPSHOTS.equals(currentFieldName)) { + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + incompatibleSnapshotIds.add(SnapshotId.fromXContent(parser)); + } + } else { + throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]"); + } + } else { + throw new ElasticsearchParseException("unknown field name [" + currentFieldName + "]"); + } + } + } else { + throw new ElasticsearchParseException("start object expected"); + } + return new RepositoryData(this.genId, this.snapshotIds, this.indexSnapshots, incompatibleSnapshotIds); } } diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 6935d277eed..0ac2d7bade1 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -128,8 +128,9 @@ import static java.util.Collections.unmodifiableMap; *
  * {@code
  *   STORE_ROOT
- *   |- index-N           - list of all snapshot name as JSON array, N is the generation of the file
+ *   |- index-N           - list of all snapshot ids and the indices belonging to each snapshot, N is the generation of the file
  *   |- index.latest      - contains the numeric value of the latest generation of the index file (i.e. N from above)
+ *   |- incompatible-snapshots - list of all snapshot ids that are no longer compatible with the current version of the cluster
  *   |- snap-20131010 - JSON serialized Snapshot for snapshot "20131010"
  *   |- meta-20131010.dat - JSON serialized MetaData for snapshot "20131010" (includes only global metadata)
  *   |- snap-20131011 - JSON serialized Snapshot for snapshot "20131011"
@@ -181,6 +182,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String INDEX_LATEST_BLOB = "index.latest";
 
+    private static final String INCOMPATIBLE_SNAPSHOTS_BLOB = "incompatible-snapshots";
+
     private static final String TESTS_FILE = "tests-";
 
     private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
@@ -232,11 +235,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = metadata.settings().getAsBoolean("readonly", false);
+
         indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
             BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, isCompress());
         indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT,
             BlobStoreIndexShardSnapshots::fromXContent, namedXContentRegistry, isCompress());
-
     }
 
     @Override
@@ -305,7 +308,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         try {
             final String snapshotName = snapshotId.getName();
             // check if the snapshot name already exists in the repository
-            if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
+            final RepositoryData repositoryData = getRepositoryData();
+            if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                 throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with the same name already exists");
             }
             if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
@@ -480,10 +484,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    public List getSnapshots() {
-        return getRepositoryData().getSnapshotIds();
-    }
-
     @Override
     public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List indices) throws IOException {
         return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
@@ -491,6 +491,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
+        if (getRepositoryData().getIncompatibleSnapshotIds().contains(snapshotId)) {
+            // an incompatible snapshot - cannot read its snapshot metadata file, just return
+            // a SnapshotInfo indicating its incompatible
+            return SnapshotInfo.incompatible(snapshotId);
+        }
+        return getSnapshotInfoInternal(snapshotId);
+    }
+
+    private SnapshotInfo getSnapshotInfoInternal(final SnapshotId snapshotId) {
         try {
             return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
         } catch (NoSuchFileException ex) {
@@ -633,9 +642,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 Streams.copy(blob, out);
                 // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
                 try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
-                    repositoryData = RepositoryData.fromXContent(parser, indexGen);
+                    repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
                 }
             }
+
+            // now load the incompatible snapshot ids, if they exist
+            try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
+                BytesStreamOutput out = new BytesStreamOutput();
+                Streams.copy(blob, out);
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                    repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
+                }
+            } catch (NoSuchFileException e) {
+                logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely reason is that " +
+                             "there are no incompatible snapshots in the repository", metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
+            }
             return repositoryData;
         } catch (NoSuchFileException ex) {
             // repository doesn't have an index blob, its a new blank repo
@@ -674,7 +695,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
             try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
-                repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
                 builder.close();
             }
             snapshotsBytes = bStream.bytes();
@@ -687,10 +708,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
                 snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
             }
-            // delete the old index file (non-generational) if it exists
-            if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
-                snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
-            }
         }
 
         // write the current generation to the index-latest file
@@ -705,6 +722,26 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         writeAtomic(INDEX_LATEST_BLOB, genBytes);
     }
 
+    /**
+     * Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository.
+     *
+     * Package private for testing.
+     */
+    void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
+        assert isReadOnly() == false; // can not write to a read only repository
+        final BytesReference bytes;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
+                repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
+                builder.close();
+            }
+            bytes = bStream.bytes();
+        }
+        // write the incompatible snapshots blob
+        writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes);
+    }
+
     /**
      * Get the latest snapshot index blob id.  Snapshot index blobs are named index-N, where N is
      * the next version number from when the index blob was written.  Each individual index-N blob is
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index ac01bc6fc5d..1d7843e7826 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -176,6 +176,11 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
             // Read snapshot info and metadata from the repository
             Repository repository = repositoriesService.repository(request.repositoryName);
             final RepositoryData repositoryData = repository.getRepositoryData();
+            final Optional incompatibleSnapshotId =
+                repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> request.snapshotName.equals(s.getName())).findFirst();
+            if (incompatibleSnapshotId.isPresent()) {
+                throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "cannot restore incompatible snapshot");
+            }
             final Optional matchingSnapshotId = repositoryData.getSnapshotIds().stream()
                 .filter(s -> request.snapshotName.equals(s.getName())).findFirst();
             if (matchingSnapshotId.isPresent() == false) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index bf65ad603a5..519393d49e1 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -21,6 +21,7 @@ package org.elasticsearch.snapshots;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -67,6 +68,8 @@ public final class SnapshotInfo implements Comparable, ToXContent,
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
 
+    private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0_UNRELEASED;
+
     private final SnapshotId snapshotId;
 
     private final SnapshotState state;
@@ -83,6 +86,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
 
     private final int successfulShards;
 
+    @Nullable
     private final Version version;
 
     private final List shardFailures;
@@ -138,7 +142,21 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         } else {
             shardFailures = Collections.emptyList();
         }
-        version = Version.readVersion(in);
+        if (in.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            version = Version.readVersion(in);
+        } else {
+            version = in.readBoolean() ? Version.readVersion(in) : null;
+        }
+    }
+
+    /**
+     * Gets a new {@link SnapshotInfo} instance for a snapshot that is incompatible with the
+     * current version of the cluster.
+     */
+    public static SnapshotInfo incompatible(SnapshotId snapshotId) {
+        return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
+                                "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
+                                null, 0L, 0L, 0, 0, Collections.emptyList());
     }
 
     /**
@@ -234,10 +252,12 @@ public final class SnapshotInfo implements Comparable, ToXContent,
     }
 
     /**
-     * Returns the version of elasticsearch that the snapshot was created with
+     * Returns the version of elasticsearch that the snapshot was created with.  Will only
+     * return {@code null} if {@link #state()} returns {@link SnapshotState#INCOMPATIBLE}.
      *
      * @return version of elasticsearch that the snapshot was created with
      */
+    @Nullable
     public Version version() {
         return version;
     }
@@ -305,8 +325,12 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         builder.startObject();
         builder.field(SNAPSHOT, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
-        builder.field(VERSION_ID, version.id);
-        builder.field(VERSION, version.toString());
+        if (version != null) {
+            builder.field(VERSION_ID, version.id);
+            builder.field(VERSION, version.toString());
+        } else {
+            builder.field(VERSION, "unknown");
+        }
         builder.startArray(INDICES);
         for (String index : indices) {
             builder.value(index);
@@ -345,6 +369,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         builder.startObject(SNAPSHOT);
         builder.field(NAME, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
+        assert version != null : "version must always be known when writing a snapshot metadata blob";
         builder.field(VERSION_ID, version.id);
         builder.startArray(INDICES);
         for (String index : indices) {
@@ -471,7 +496,11 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         for (String index : indices) {
             out.writeString(index);
         }
-        out.writeByte(state.value());
+        if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
+            out.writeByte(SnapshotState.FAILED.value());
+        } else {
+            out.writeByte(state.value());
+        }
         out.writeOptionalString(reason);
         out.writeVLong(startTime);
         out.writeVLong(endTime);
@@ -481,7 +510,20 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         for (SnapshotShardFailure failure : shardFailures) {
             failure.writeTo(out);
         }
-        Version.writeVersion(version, out);
+        if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            Version versionToWrite = version;
+            if (versionToWrite == null) {
+                versionToWrite = Version.CURRENT;
+            }
+            Version.writeVersion(versionToWrite, out);
+        } else {
+            if (version != null) {
+                out.writeBoolean(true);
+                Version.writeVersion(version, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
     }
 
     private static SnapshotState snapshotState(final String reason, final List shardFailures) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
index b893a372d13..3df5f8fff04 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
@@ -39,7 +39,11 @@ public enum SnapshotState {
     /**
      * Snapshot was partial successful
      */
-    PARTIAL((byte) 3, true, true);
+    PARTIAL((byte) 3, true, true),
+    /**
+     * Snapshot is incompatible with the current version of the cluster
+     */
+    INCOMPATIBLE((byte) 4, true, false);
 
     private byte value;
 
@@ -47,7 +51,7 @@ public enum SnapshotState {
 
     private boolean restorable;
 
-    private SnapshotState(byte value, boolean completed, boolean restorable) {
+    SnapshotState(byte value, boolean completed, boolean restorable) {
         this.value = value;
         this.completed = completed;
         this.restorable = restorable;
@@ -97,6 +101,8 @@ public enum SnapshotState {
                 return FAILED;
             case 3:
                 return PARTIAL;
+            case 4:
+                return INCOMPATIBLE;
             default:
                 throw new IllegalArgumentException("No snapshot state for value [" + value + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 1b5bfde167c..2f93c20c37a 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -131,15 +131,15 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     }
 
     /**
-     * Retrieves list of snapshot ids that are present in a repository
+     * Gets the {@link RepositoryData} for the given repository.
      *
      * @param repositoryName repository name
-     * @return list of snapshot ids
+     * @return repository data
      */
-    public List snapshotIds(final String repositoryName) {
+    public RepositoryData getRepositoryData(final String repositoryName) {
         Repository repository = repositoriesService.repository(repositoryName);
         assert repository != null; // should only be called once we've validated the repository exists
-        return repository.getRepositoryData().getSnapshotIds();
+        return repository.getRepositoryData();
     }
 
     /**
@@ -1004,6 +1004,11 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         // First, look for the snapshot in the repository
         final Repository repository = repositoriesService.repository(repositoryName);
         final RepositoryData repositoryData = repository.getRepositoryData();
+        final Optional incompatibleSnapshotId =
+            repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
+        if (incompatibleSnapshotId.isPresent()) {
+            throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
+        }
         Optional matchedEntry = repositoryData.getSnapshotIds()
                                                 .stream()
                                                 .filter(s -> s.getName().equals(snapshotName))
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index d9d6015e422..ce3fb4c129c 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -126,6 +126,7 @@ import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -1493,7 +1494,7 @@ public class IndexShardTests extends IndexShardTestCase {
         public RepositoryData getRepositoryData() {
             Map> map = new HashMap<>();
             map.put(new IndexId(indexName, "blah"), emptySet());
-            return RepositoryData.initRepositoryData(Collections.emptyList(), map);
+            return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), map, Collections.emptyList());
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
index 97d415fe4f9..f9c620e3b9e 100644
--- a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
@@ -37,6 +37,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.greaterThan;
 
 /**
@@ -54,10 +55,10 @@ public class RepositoryDataTests extends ESTestCase {
     public void testXContent() throws IOException {
         RepositoryData repositoryData = generateRandomRepoData();
         XContentBuilder builder = JsonXContent.contentBuilder();
-        repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         long gen = (long) randomIntBetween(0, 500);
-        RepositoryData fromXContent = RepositoryData.fromXContent(parser, gen);
+        RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen);
         assertEquals(repositoryData, fromXContent);
         assertEquals(gen, fromXContent.getGenId());
     }
@@ -65,7 +66,6 @@ public class RepositoryDataTests extends ESTestCase {
     public void testAddSnapshots() {
         RepositoryData repositoryData = generateRandomRepoData();
         // test that adding the same snapshot id to the repository data throws an exception
-        final SnapshotId snapshotId = repositoryData.getSnapshotIds().get(0);
         Map indexIdMap = repositoryData.getIndices();
         // test that adding a snapshot and its indices works
         SnapshotId newSnapshot = new SnapshotId(randomAsciiOfLength(7), UUIDs.randomBase64UUID());
@@ -95,6 +95,22 @@ public class RepositoryDataTests extends ESTestCase {
         assertEquals(repositoryData.getGenId(), newRepoData.getGenId());
     }
 
+    public void testInitIndices() {
+        final int numSnapshots = randomIntBetween(1, 30);
+        final List snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), Collections.emptyList());
+        // test that initializing indices works
+        Map> indices = randomIndices(snapshotIds);
+        RepositoryData newRepoData = repositoryData.initIndices(indices);
+        assertEquals(repositoryData.getSnapshotIds(), newRepoData.getSnapshotIds());
+        for (IndexId indexId : indices.keySet()) {
+            assertEquals(indices.get(indexId), newRepoData.getSnapshots(indexId));
+        }
+    }
+
     public void testRemoveSnapshot() {
         RepositoryData repositoryData = generateRandomRepoData();
         List snapshotIds = new ArrayList<>(repositoryData.getSnapshotIds());
@@ -121,8 +137,12 @@ public class RepositoryDataTests extends ESTestCase {
     }
 
     public static RepositoryData generateRandomRepoData() {
-        List snapshotIds = randomSnapshots(new ArrayList<>());
-        return RepositoryData.initRepositoryData(snapshotIds, randomIndices(snapshotIds));
+        return generateRandomRepoData(new ArrayList<>());
+    }
+
+    public static RepositoryData generateRandomRepoData(final List origSnapshotIds) {
+        List snapshotIds = randomSnapshots(origSnapshotIds);
+        return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, randomIndices(snapshotIds), Collections.emptyList());
     }
 
     private static List randomSnapshots(final List origSnapshotIds) {
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index f5f036a2359..6e538e721a4 100644
--- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -36,6 +36,7 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -95,9 +96,8 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
 
-        List snapshotIds = repository.getSnapshots().stream()
-                                                             .sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
-                                                             .collect(Collectors.toList());
+        List snapshotIds = repository.getRepositoryData().getSnapshotIds().stream()
+            .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
         assertThat(snapshotIds, equalTo(originalSnapshots));
     }
 
@@ -105,7 +105,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         final BlobStoreRepository repository = setupRepo();
 
         // write to and read from a index file with no entries
-        assertThat(repository.getSnapshots().size(), equalTo(0));
+        assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0));
         final RepositoryData emptyData = RepositoryData.EMPTY;
         repository.writeIndexGen(emptyData, emptyData.getGenId());
         RepositoryData repoData = repository.getRepositoryData();
@@ -162,6 +162,33 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         expectThrows(RepositoryException.class, () -> repository.writeIndexGen(repositoryData, repositoryData.getGenId()));
     }
 
+    public void testReadAndWriteIncompatibleSnapshots() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to and read from incompatible snapshots file with no entries
+        assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
+        RepositoryData emptyData = RepositoryData.EMPTY;
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        repository.writeIncompatibleSnapshots(emptyData);
+        RepositoryData readData = repository.getRepositoryData();
+        assertEquals(emptyData, readData);
+        assertEquals(0, readData.getIndices().size());
+        assertEquals(0, readData.getSnapshotIds().size());
+
+        // write to and read from incompatible snapshots with some number of entries
+        final int numSnapshots = randomIntBetween(1, 20);
+        final List snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        RepositoryData repositoryData = new RepositoryData(readData.getGenId(), Collections.emptyList(), Collections.emptyMap(),
+                                                              snapshotIds);
+        repository.blobContainer().deleteBlob("incompatible-snapshots");
+        repository.writeIncompatibleSnapshots(repositoryData);
+        readData = repository.getRepositoryData();
+        assertEquals(repositoryData.getIncompatibleSnapshotIds(), readData.getIncompatibleSnapshotIds());
+    }
+
     private BlobStoreRepository setupRepo() {
         final Client client = client();
         final Path location = ESIntegTestCase.randomRepoPath(node().settings());
diff --git a/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip
new file mode 100644
index 00000000000..9edf7d57527
Binary files /dev/null and b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip differ