honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Gracefully handles pre 2.x compressed snapshots 

In pre 2.x versions, if the repository was set to compress snapshots,
then snapshots would be compressed with the LZF algorithm.  In 5.x,
Elasticsearch no longer supports the LZF compression algorithm.  This
presents an issue when retrieving snapshots in a repository or upgrading
repository data to the 5.x version, because Elasticsearch throws an
exception when it tries to read the snapshot metadata because it was
compressed using LZF.

This commit gracefully handles the situation by introducing a new
incompatible-snapshots blob to the repository.  For any pre-2.x snapshot
that cannot be read, that snapshot is removed from the list of active
snapshots, because the snapshot could not be restored anyway.  Instead,
the snapshot is recorded in the incompatible-snapshots blob.  When
listing snapshots, both active snapshots and incompatible snapshots will
be listed, with incompatible snapshots showing a `INCOMPATIBLE` state.
Any attempt to restore an incompatible snapshot will result in an
exception.
This commit is contained in:
Ali Beyad 2017-01-06 17:48:57 -05:00
parent ded694fc83
commit b0c009ae76
12 changed files with 297 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
View File

@ -88,7 +88,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
currentSnapshotIds.add(snapshotId);
}
if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
for (SnapshotId snapshotId : snapshotsService.snapshotIds(repository)) {
for (SnapshotId snapshotId : snapshotsService.getRepositoryData(repository).getAllSnapshotIds()) {
allSnapshotIds.put(snapshotId.getName(), snapshotId);
}
}

9
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
View File

@ -36,7 +36,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.snapshots.Snapshot;
import org.elasticsearch.snapshots.SnapshotException;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotMissingException;
@ -201,7 +203,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
final String repositoryName = request.repository();
if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
final Map<String, SnapshotId> matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream()
final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName);
final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream()
.filter(s -> requestedSnapshotNames.contains(s.getName()))
.collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
for (final String snapshotName : request.snapshots()) {
@ -220,6 +223,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
} else {
throw new SnapshotMissingException(repositoryName, snapshotName);
}
} else if (repositoryData.getIncompatibleSnapshotIds().contains(snapshotId)) {
throw new SnapshotException(repositoryName, snapshotName, "cannot get the status for an incompatible snapshot");
}
SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
@ -243,7 +248,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
default:
throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
}
builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotInfo.snapshotId()), state, Collections.unmodifiableList(shardStatusBuilder)));
builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
}
}
}

130
core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View File

@ -42,7 +42,7 @@ import java.util.stream.Collectors;
* A class that represents the data in a repository, as captured in the
* repository's index blob.
*/
public final class RepositoryData implements ToXContent {
public final class RepositoryData {
/**
* The generation value indicating the repository has no index generational files.
@ -51,7 +51,8 @@ public final class RepositoryData implements ToXContent {
/**
* An instance initialized for an empty repository.
*/
public static final RepositoryData EMPTY = new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), Collections.emptyMap());
public static final RepositoryData EMPTY =
new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), Collections.emptyMap(), Collections.emptyList());
/**
* The generational id of the index file from which the repository data was read.
@ -69,25 +70,24 @@ public final class RepositoryData implements ToXContent {
* The snapshots that each index belongs to.
*/
private final Map<IndexId, Set<SnapshotId>> indexSnapshots;
/**
* The snapshots that are no longer compatible with the current cluster ES version.
*/
private final List<SnapshotId> incompatibleSnapshotIds;
private RepositoryData(long genId, List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots) {
public RepositoryData(long genId, List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots,
List<SnapshotId> incompatibleSnapshotIds) {
this.genId = genId;
this.snapshotIds = Collections.unmodifiableList(snapshotIds);
this.indices = Collections.unmodifiableMap(indexSnapshots.keySet()
.stream()
.collect(Collectors.toMap(IndexId::getName, Function.identity())));
this.indexSnapshots = Collections.unmodifiableMap(indexSnapshots);
}
/**
* Creates an instance of {@link RepositoryData} on a fresh repository (one that has no index-N files).
*/
public static RepositoryData initRepositoryData(List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots) {
return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, indexSnapshots);
this.incompatibleSnapshotIds = Collections.unmodifiableList(incompatibleSnapshotIds);
}
protected RepositoryData copy() {
return new RepositoryData(genId, snapshotIds, indexSnapshots);
return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
}
/**
@ -104,6 +104,25 @@ public final class RepositoryData implements ToXContent {
return snapshotIds;
}
/**
* Returns an immutable collection of the snapshot ids in the repository that are incompatible with the
* current ES version.
*/
public List<SnapshotId> getIncompatibleSnapshotIds() {
return incompatibleSnapshotIds;
}
/**
* Returns an immutable collection of all the snapshot ids in the repository, both active and
* incompatible snapshots.
*/
public List<SnapshotId> getAllSnapshotIds() {
List<SnapshotId> allSnapshotIds = new ArrayList<>(snapshotIds.size() + incompatibleSnapshotIds.size());
allSnapshotIds.addAll(snapshotIds);
allSnapshotIds.addAll(incompatibleSnapshotIds);
return Collections.unmodifiableList(allSnapshotIds);
}
/**
* Returns an unmodifiable map of the index names to {@link IndexId} in the repository.
*/
@ -139,7 +158,7 @@ public final class RepositoryData implements ToXContent {
allIndexSnapshots.put(indexId, ids);
}
}
return new RepositoryData(genId, snapshots, allIndexSnapshots);
return new RepositoryData(genId, snapshots, allIndexSnapshots, incompatibleSnapshotIds);
}
/**
@ -168,7 +187,21 @@ public final class RepositoryData implements ToXContent {
indexSnapshots.put(indexId, set);
}
return new RepositoryData(genId, newSnapshotIds, indexSnapshots);
return new RepositoryData(genId, newSnapshotIds, indexSnapshots, incompatibleSnapshotIds);
}
/**
* Returns a new {@link RepositoryData} instance containing the same snapshot data as the
* invoking instance, with the given incompatible snapshots added to the new instance.
*/
public RepositoryData addIncompatibleSnapshots(final List<SnapshotId> incompatibleSnapshotIds) {
List<SnapshotId> newSnapshotIds = new ArrayList<>(this.snapshotIds);
List<SnapshotId> newIncompatibleSnapshotIds = new ArrayList<>(this.incompatibleSnapshotIds);
for (SnapshotId snapshotId : incompatibleSnapshotIds) {
newSnapshotIds.remove(snapshotId);
newIncompatibleSnapshotIds.add(snapshotId);
}
return new RepositoryData(this.genId, newSnapshotIds, this.indexSnapshots, newIncompatibleSnapshotIds);
}
/**
@ -182,6 +215,13 @@ public final class RepositoryData implements ToXContent {
return snapshotIds;
}
/**
* Initializes the indices in the repository metadata; returns a new instance.
*/
public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
@ -193,12 +233,13 @@ public final class RepositoryData implements ToXContent {
@SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj;
return snapshotIds.equals(that.snapshotIds)
&& indices.equals(that.indices)
&& indexSnapshots.equals(that.indexSnapshots);
&& indexSnapshots.equals(that.indexSnapshots)
&& incompatibleSnapshotIds.equals(that.incompatibleSnapshotIds);
}
@Override
public int hashCode() {
return Objects.hash(snapshotIds, indices, indexSnapshots);
return Objects.hash(snapshotIds, indices, indexSnapshots, incompatibleSnapshotIds);
}
/**
@ -247,11 +288,15 @@ public final class RepositoryData implements ToXContent {
}
private static final String SNAPSHOTS = "snapshots";
private static final String INCOMPATIBLE_SNAPSHOTS = "incompatible-snapshots";
private static final String INDICES = "indices";
private static final String INDEX_ID = "id";
@Override
public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
/**
* Writes the snapshots metadata and the related indices metadata to x-content, omitting the
* incompatible snapshots.
*/
public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
builder.startObject();
// write the snapshots list
builder.startArray(SNAPSHOTS);
@ -278,7 +323,10 @@ public final class RepositoryData implements ToXContent {
return builder;
}
public static RepositoryData fromXContent(final XContentParser parser, final long genId) throws IOException {
/**
* Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
*/
public static RepositoryData snapshotsFromXContent(final XContentParser parser, long genId) throws IOException {
List<SnapshotId> snapshots = new ArrayList<>();
Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
@ -327,7 +375,51 @@ public final class RepositoryData implements ToXContent {
} else {
throw new ElasticsearchParseException("start object expected");
}
return new RepositoryData(genId, snapshots, indexSnapshots);
return new RepositoryData(genId, snapshots, indexSnapshots, Collections.emptyList());
}
/**
* Writes the incompatible snapshot ids to x-content.
*/
public XContentBuilder incompatibleSnapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params)
throws IOException {
builder.startObject();
// write the incompatible snapshots list
builder.startArray(INCOMPATIBLE_SNAPSHOTS);
for (final SnapshotId snapshot : getIncompatibleSnapshotIds()) {
snapshot.toXContent(builder, params);
}
builder.endArray();
builder.endObject();
return builder;
}
/**
* Reads the incompatible snapshot ids from x-content, loading them into a new instance of {@link RepositoryData}
* that is created from the invoking instance, plus the incompatible snapshots that are read from x-content.
*/
public RepositoryData incompatibleSnapshotsFromXContent(final XContentParser parser) throws IOException {
List<SnapshotId> incompatibleSnapshotIds = new ArrayList<>();
if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
if (INCOMPATIBLE_SNAPSHOTS.equals(currentFieldName)) {
if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
incompatibleSnapshotIds.add(SnapshotId.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]");
}
} else {
throw new ElasticsearchParseException("unknown field name [" + currentFieldName + "]");
}
}
} else {
throw new ElasticsearchParseException("start object expected");
}
return new RepositoryData(this.genId, this.snapshotIds, this.indexSnapshots, incompatibleSnapshotIds);
}
}

63
core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -128,8 +128,9 @@ import static java.util.Collections.unmodifiableMap;
* <pre>
* {@code
* STORE_ROOT
* |- index-N - list of all snapshot name as JSON array, N is the generation of the file
* |- index-N - list of all snapshot ids and the indices belonging to each snapshot, N is the generation of the file
* |- index.latest - contains the numeric value of the latest generation of the index file (i.e. N from above)
* |- incompatible-snapshots - list of all snapshot ids that are no longer compatible with the current version of the cluster
* |- snap-20131010 - JSON serialized Snapshot for snapshot "20131010"
* |- meta-20131010.dat - JSON serialized MetaData for snapshot "20131010" (includes only global metadata)
* |- snap-20131011 - JSON serialized Snapshot for snapshot "20131011"
@ -181,6 +182,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
private static final String INDEX_LATEST_BLOB = "index.latest";
private static final String INCOMPATIBLE_SNAPSHOTS_BLOB = "incompatible-snapshots";
private static final String TESTS_FILE = "tests-";
private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
@ -232,11 +235,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
readOnly = metadata.settings().getAsBoolean("readonly", false);
indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, isCompress());
indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT,
BlobStoreIndexShardSnapshots::fromXContent, namedXContentRegistry, isCompress());
}
@Override
@ -305,7 +308,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
try {
final String snapshotName = snapshotId.getName();
// check if the snapshot name already exists in the repository
if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
final RepositoryData repositoryData = getRepositoryData();
if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with the same name already exists");
}
if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
@ -480,10 +484,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
}
}
public List<SnapshotId> getSnapshots() {
return getRepositoryData().getSnapshotIds();
}
@Override
public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
@ -491,6 +491,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
@Override
public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
if (getRepositoryData().getIncompatibleSnapshotIds().contains(snapshotId)) {
// an incompatible snapshot - cannot read its snapshot metadata file, just return
// a SnapshotInfo indicating its incompatible
return SnapshotInfo.incompatible(snapshotId);
}
return getSnapshotInfoInternal(snapshotId);
}
private SnapshotInfo getSnapshotInfoInternal(final SnapshotId snapshotId) {
try {
return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {
@ -633,9 +642,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
Streams.copy(blob, out);
// EMPTY is safe here because RepositoryData#fromXContent calls namedObject
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
repositoryData = RepositoryData.fromXContent(parser, indexGen);
repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
}
}
// now load the incompatible snapshot ids, if they exist
try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
BytesStreamOutput out = new BytesStreamOutput();
Streams.copy(blob, out);
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
}
} catch (NoSuchFileException e) {
logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely reason is that " +
"there are no incompatible snapshots in the repository", metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
}
return repositoryData;
} catch (NoSuchFileException ex) {
// repository doesn't have an index blob, its a new blank repo
@ -674,7 +695,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
}
snapshotsBytes = bStream.bytes();
@ -687,10 +708,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
}
// delete the old index file (non-generational) if it exists
if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
}
}
// write the current generation to the index-latest file
@ -705,6 +722,26 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
writeAtomic(INDEX_LATEST_BLOB, genBytes);
}
/**
* Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository.
*
* Package private for testing.
*/
void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
assert isReadOnly() == false; // can not write to a read only repository
final BytesReference bytes;
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
}
bytes = bStream.bytes();
}
// write the incompatible snapshots blob
writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes);
}
/**
* Get the latest snapshot index blob id. Snapshot index blobs are named index-N, where N is
* the next version number from when the index blob was written. Each individual index-N blob is

5
core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View File

@ -176,6 +176,11 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
// Read snapshot info and metadata from the repository
Repository repository = repositoriesService.repository(request.repositoryName);
final RepositoryData repositoryData = repository.getRepositoryData();
final Optional<SnapshotId> incompatibleSnapshotId =
repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> request.snapshotName.equals(s.getName())).findFirst();
if (incompatibleSnapshotId.isPresent()) {
throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "cannot restore incompatible snapshot");
}
final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds().stream()
.filter(s -> request.snapshotName.equals(s.getName())).findFirst();
if (matchingSnapshotId.isPresent() == false) {

54
core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.snapshots;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -67,6 +68,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
private static final String TOTAL_SHARDS = "total_shards";
private static final String SUCCESSFUL_SHARDS = "successful_shards";
private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0_UNRELEASED;
private final SnapshotId snapshotId;
private final SnapshotState state;
@ -83,6 +86,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
private final int successfulShards;
@Nullable
private final Version version;
private final List<SnapshotShardFailure> shardFailures;
@ -138,7 +142,21 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
} else {
shardFailures = Collections.emptyList();
}
version = Version.readVersion(in);
if (in.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
version = Version.readVersion(in);
} else {
version = in.readBoolean() ? Version.readVersion(in) : null;
}
}
/**
* Gets a new {@link SnapshotInfo} instance for a snapshot that is incompatible with the
* current version of the cluster.
*/
public static SnapshotInfo incompatible(SnapshotId snapshotId) {
return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
"the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
null, 0L, 0L, 0, 0, Collections.emptyList());
}
/**
@ -234,10 +252,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns the version of elasticsearch that the snapshot was created with
* Returns the version of elasticsearch that the snapshot was created with. Will only
* return {@code null} if {@link #state()} returns {@link SnapshotState#INCOMPATIBLE}.
*
* @return version of elasticsearch that the snapshot was created with
*/
@Nullable
public Version version() {
return version;
}
@ -305,8 +325,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
builder.startObject();
builder.field(SNAPSHOT, snapshotId.getName());
builder.field(UUID, snapshotId.getUUID());
builder.field(VERSION_ID, version.id);
builder.field(VERSION, version.toString());
if (version != null) {
builder.field(VERSION_ID, version.id);
builder.field(VERSION, version.toString());
} else {
builder.field(VERSION, "unknown");
}
builder.startArray(INDICES);
for (String index : indices) {
builder.value(index);
@ -345,6 +369,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
builder.startObject(SNAPSHOT);
builder.field(NAME, snapshotId.getName());
builder.field(UUID, snapshotId.getUUID());
assert version != null : "version must always be known when writing a snapshot metadata blob";
builder.field(VERSION_ID, version.id);
builder.startArray(INDICES);
for (String index : indices) {
@ -471,7 +496,11 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
for (String index : indices) {
out.writeString(index);
}
out.writeByte(state.value());
if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
out.writeByte(SnapshotState.FAILED.value());
} else {
out.writeByte(state.value());
}
out.writeOptionalString(reason);
out.writeVLong(startTime);
out.writeVLong(endTime);
@ -481,7 +510,20 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
for (SnapshotShardFailure failure : shardFailures) {
failure.writeTo(out);
}
Version.writeVersion(version, out);
if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
Version versionToWrite = version;
if (versionToWrite == null) {
versionToWrite = Version.CURRENT;
}
Version.writeVersion(versionToWrite, out);
} else {
if (version != null) {
out.writeBoolean(true);
Version.writeVersion(version, out);
} else {
out.writeBoolean(false);
}
}
}
private static SnapshotState snapshotState(final String reason, final List<SnapshotShardFailure> shardFailures) {

10
core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
View File

@ -39,7 +39,11 @@ public enum SnapshotState {
/**
* Snapshot was partial successful
*/
PARTIAL((byte) 3, true, true);
PARTIAL((byte) 3, true, true),
/**
* Snapshot is incompatible with the current version of the cluster
*/
INCOMPATIBLE((byte) 4, true, false);
private byte value;
@ -47,7 +51,7 @@ public enum SnapshotState {
private boolean restorable;
private SnapshotState(byte value, boolean completed, boolean restorable) {
SnapshotState(byte value, boolean completed, boolean restorable) {
this.value = value;
this.completed = completed;
this.restorable = restorable;
@ -97,6 +101,8 @@ public enum SnapshotState {
return FAILED;
case 3:
return PARTIAL;
case 4:
return INCOMPATIBLE;
default:
throw new IllegalArgumentException("No snapshot state for value [" + value + "]");
}

13
core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
View File

@ -131,15 +131,15 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
}
/**
* Retrieves list of snapshot ids that are present in a repository
* Gets the {@link RepositoryData} for the given repository.
*
* @param repositoryName repository name
* @return list of snapshot ids
* @return repository data
*/
public List<SnapshotId> snapshotIds(final String repositoryName) {
public RepositoryData getRepositoryData(final String repositoryName) {
Repository repository = repositoriesService.repository(repositoryName);
assert repository != null; // should only be called once we've validated the repository exists
return repository.getRepositoryData().getSnapshotIds();
return repository.getRepositoryData();
}
/**
@ -1004,6 +1004,11 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
// First, look for the snapshot in the repository
final Repository repository = repositoriesService.repository(repositoryName);
final RepositoryData repositoryData = repository.getRepositoryData();
final Optional<SnapshotId> incompatibleSnapshotId =
repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
if (incompatibleSnapshotId.isPresent()) {
throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
}
Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds()
.stream()
.filter(s -> s.getName().equals(snapshotName))

3
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -126,6 +126,7 @@ import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex;
import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
@ -1493,7 +1494,7 @@ public class IndexShardTests extends IndexShardTestCase {
public RepositoryData getRepositoryData() {
Map<IndexId, Set<SnapshotId>> map = new HashMap<>();
map.put(new IndexId(indexName, "blah"), emptySet());
return RepositoryData.initRepositoryData(Collections.emptyList(), map);
return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), map, Collections.emptyList());
}
@Override

30
core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
View File

@ -37,6 +37,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
import static org.hamcrest.Matchers.greaterThan;
/**
@ -54,10 +55,10 @@ public class RepositoryDataTests extends ESTestCase {
public void testXContent() throws IOException {
RepositoryData repositoryData = generateRandomRepoData();
XContentBuilder builder = JsonXContent.contentBuilder();
repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
long gen = (long) randomIntBetween(0, 500);
RepositoryData fromXContent = RepositoryData.fromXContent(parser, gen);
RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen);
assertEquals(repositoryData, fromXContent);
assertEquals(gen, fromXContent.getGenId());
}
@ -65,7 +66,6 @@ public class RepositoryDataTests extends ESTestCase {
public void testAddSnapshots() {
RepositoryData repositoryData = generateRandomRepoData();
// test that adding the same snapshot id to the repository data throws an exception
final SnapshotId snapshotId = repositoryData.getSnapshotIds().get(0);
Map<String, IndexId> indexIdMap = repositoryData.getIndices();
// test that adding a snapshot and its indices works
SnapshotId newSnapshot = new SnapshotId(randomAsciiOfLength(7), UUIDs.randomBase64UUID());
@ -95,6 +95,22 @@ public class RepositoryDataTests extends ESTestCase {
assertEquals(repositoryData.getGenId(), newRepoData.getGenId());
}
public void testInitIndices() {
final int numSnapshots = randomIntBetween(1, 30);
final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
for (int i = 0; i < numSnapshots; i++) {
snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
}
RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), Collections.emptyList());
// test that initializing indices works
Map<IndexId, Set<SnapshotId>> indices = randomIndices(snapshotIds);
RepositoryData newRepoData = repositoryData.initIndices(indices);
assertEquals(repositoryData.getSnapshotIds(), newRepoData.getSnapshotIds());
for (IndexId indexId : indices.keySet()) {
assertEquals(indices.get(indexId), newRepoData.getSnapshots(indexId));
}
}
public void testRemoveSnapshot() {
RepositoryData repositoryData = generateRandomRepoData();
List<SnapshotId> snapshotIds = new ArrayList<>(repositoryData.getSnapshotIds());
@ -121,8 +137,12 @@ public class RepositoryDataTests extends ESTestCase {
}
public static RepositoryData generateRandomRepoData() {
List<SnapshotId> snapshotIds = randomSnapshots(new ArrayList<>());
return RepositoryData.initRepositoryData(snapshotIds, randomIndices(snapshotIds));
return generateRandomRepoData(new ArrayList<>());
}
public static RepositoryData generateRandomRepoData(final List<SnapshotId> origSnapshotIds) {
List<SnapshotId> snapshotIds = randomSnapshots(origSnapshotIds);
return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, randomIndices(snapshotIds), Collections.emptyList());
}
private static List<SnapshotId> randomSnapshots(final List<SnapshotId> origSnapshotIds) {

35
core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
View File

@ -36,6 +36,7 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -95,9 +96,8 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
(BlobStoreRepository) repositoriesService.repository(repositoryName);
final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
List<SnapshotId> snapshotIds = repository.getSnapshots().stream()
.sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
.collect(Collectors.toList());
List<SnapshotId> snapshotIds = repository.getRepositoryData().getSnapshotIds().stream()
.sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
assertThat(snapshotIds, equalTo(originalSnapshots));
}
@ -105,7 +105,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
final BlobStoreRepository repository = setupRepo();
// write to and read from a index file with no entries
assertThat(repository.getSnapshots().size(), equalTo(0));
assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0));
final RepositoryData emptyData = RepositoryData.EMPTY;
repository.writeIndexGen(emptyData, emptyData.getGenId());
RepositoryData repoData = repository.getRepositoryData();
@ -162,6 +162,33 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
expectThrows(RepositoryException.class, () -> repository.writeIndexGen(repositoryData, repositoryData.getGenId()));
}
public void testReadAndWriteIncompatibleSnapshots() throws Exception {
final BlobStoreRepository repository = setupRepo();
// write to and read from incompatible snapshots file with no entries
assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
RepositoryData emptyData = RepositoryData.EMPTY;
repository.writeIndexGen(emptyData, emptyData.getGenId());
repository.writeIncompatibleSnapshots(emptyData);
RepositoryData readData = repository.getRepositoryData();
assertEquals(emptyData, readData);
assertEquals(0, readData.getIndices().size());
assertEquals(0, readData.getSnapshotIds().size());
// write to and read from incompatible snapshots with some number of entries
final int numSnapshots = randomIntBetween(1, 20);
final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
for (int i = 0; i < numSnapshots; i++) {
snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
}
RepositoryData repositoryData = new RepositoryData(readData.getGenId(), Collections.emptyList(), Collections.emptyMap(),
snapshotIds);
repository.blobContainer().deleteBlob("incompatible-snapshots");
repository.writeIncompatibleSnapshots(repositoryData);
readData = repository.getRepositoryData();
assertEquals(repositoryData.getIncompatibleSnapshotIds(), readData.getIncompatibleSnapshotIds());
}
private BlobStoreRepository setupRepo() {
final Client client = client();
final Path location = ESIntegTestCase.randomRepoPath(node().settings());

BIN
core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip Normal file
View File

Binary file not shown.