diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 2654ac0c269..53b1bf86f66 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -26,10 +26,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; @@ -72,7 +72,7 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction listener) { SnapshotsService.SnapshotRequest snapshotRequest = - new SnapshotsService.SnapshotRequest("create_snapshot [" + request.snapshot() + "]", request.snapshot(), request.repository()) + new SnapshotsService.SnapshotRequest(request.repository(), request.snapshot(), "create_snapshot [" + request.snapshot() + "]") .indices(request.indices()) .indicesOptions(request.indicesOptions()) .partial(request.partial()) @@ -84,19 +84,19 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction listener) { - SnapshotId snapshotIds = new SnapshotId(request.repository(), request.snapshot()); - snapshotsService.deleteSnapshot(snapshotIds, new SnapshotsService.DeleteSnapshotListener() { + snapshotsService.deleteSnapshot(request.repository(), request.snapshot(), new SnapshotsService.DeleteSnapshotListener() { @Override public void onResponse() { listener.onResponse(new DeleteSnapshotResponse(true)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index ec996e6d366..924f5a90d42 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -42,7 +42,7 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent { } GetSnapshotsResponse(List snapshots) { - this.snapshots = snapshots; + this.snapshots = Collections.unmodifiableList(snapshots); } /** diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 833b1a62289..f734201e7bd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -26,20 +26,22 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -52,7 +54,8 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction listener) { + protected void masterOperation(final GetSnapshotsRequest request, ClusterState state, + final ActionListener listener) { try { + final String repository = request.repository(); List snapshotInfoBuilder = new ArrayList<>(); if (isAllSnapshots(request.snapshots())) { - snapshotInfoBuilder.addAll(snapshotsService.snapshots(request.repository(), request.ignoreUnavailable())); + snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository)); + snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, + snapshotsService.snapshotIds(repository), + request.ignoreUnavailable())); } else if (isCurrentSnapshots(request.snapshots())) { - snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(request.repository())); + snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository)); } else { - Set snapshotsToGet = new LinkedHashSet<>(); // to keep insertion order - List snapshots = null; + final Map allSnapshotIds = new HashMap<>(); + for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) { + SnapshotId snapshotId = snapshotInfo.snapshotId(); + allSnapshotIds.put(snapshotId.getName(), snapshotId); + } + for (SnapshotId snapshotId : snapshotsService.snapshotIds(repository)) { + allSnapshotIds.put(snapshotId.getName(), snapshotId); + } + final Set toResolve = new LinkedHashSet<>(); // maintain order for (String snapshotOrPattern : request.snapshots()) { if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { - snapshotsToGet.add(snapshotOrPattern); - } else { - if (snapshots == null) { // lazily load snapshots - snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable()); + if (allSnapshotIds.containsKey(snapshotOrPattern)) { + toResolve.add(allSnapshotIds.get(snapshotOrPattern)); + } else if (request.ignoreUnavailable() == false) { + throw new SnapshotMissingException(repository, snapshotOrPattern); } - for (SnapshotInfo snapshot : snapshots) { - if (Regex.simpleMatch(snapshotOrPattern, snapshot.name())) { - snapshotsToGet.add(snapshot.name()); + } else { + for (Map.Entry entry : allSnapshotIds.entrySet()) { + if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { + toResolve.add(entry.getValue()); } } } } - for (String snapshot : snapshotsToGet) { - SnapshotId snapshotId = new SnapshotId(request.repository(), snapshot); - snapshotInfoBuilder.add(snapshotsService.snapshot(snapshotId)); + + if (toResolve.isEmpty() && request.ignoreUnavailable() == false) { + throw new SnapshotMissingException(repository, request.snapshots()[0]); } + + snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, new ArrayList<>(toResolve), request.ignoreUnavailable())); } - listener.onResponse(new GetSnapshotsResponse(Collections.unmodifiableList(snapshotInfoBuilder))); + listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder)); } catch (Throwable t) { listener.onFailure(t); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index cbbc195370c..a38fbce46c2 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -26,12 +26,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -72,23 +72,22 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeAction listener) { - RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest( - "restore_snapshot[" + request.snapshot() + "]", request.repository(), request.snapshot(), + protected void masterOperation(final RestoreSnapshotRequest request, final ClusterState state, final ActionListener listener) { + RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(request.repository(), request.snapshot(), request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(), request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(), - request.indexSettings(), request.ignoreIndexSettings()); + request.indexSettings(), request.ignoreIndexSettings(), "restore_snapshot[" + request.snapshot() + "]"); restoreService.restoreSnapshot(restoreRequest, new ActionListener() { @Override public void onResponse(RestoreInfo restoreInfo) { if (restoreInfo == null && request.waitForCompletion()) { restoreService.addListener(new ActionListener() { - SnapshotId snapshotId = new SnapshotId(request.repository(), request.snapshot()); - @Override public void onResponse(RestoreService.RestoreCompletionResponse restoreCompletionResponse) { - if (this.snapshotId.equals(restoreCompletionResponse.getSnapshotId())) { + final Snapshot snapshot = restoreCompletionResponse.getSnapshot(); + if (snapshot.getRepository().equals(request.repository()) && + snapshot.getSnapshotId().getName().equals(request.snapshot())) { listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); restoreService.removeListener(this); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java index 54f87f3c5fc..1a5ef9ab933 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java @@ -20,7 +20,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.status; import org.elasticsearch.cluster.SnapshotsInProgress.State; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static java.util.Collections.unmodifiableMap; @@ -44,7 +45,7 @@ import static java.util.Collections.unmodifiableMap; */ public class SnapshotStatus implements ToXContent, Streamable { - private SnapshotId snapshotId; + private Snapshot snapshot; private State state; @@ -56,11 +57,10 @@ public class SnapshotStatus implements ToXContent, Streamable { private SnapshotStats stats; - - SnapshotStatus(SnapshotId snapshotId, State state, List shards) { - this.snapshotId = snapshotId; - this.state = state; - this.shards = shards; + SnapshotStatus(final Snapshot snapshot, final State state, final List shards) { + this.snapshot = Objects.requireNonNull(snapshot); + this.state = Objects.requireNonNull(state); + this.shards = Objects.requireNonNull(shards); shardsStats = new SnapshotShardsStats(shards); updateShardStats(); } @@ -69,10 +69,10 @@ public class SnapshotStatus implements ToXContent, Streamable { } /** - * Returns snapshot id + * Returns snapshot */ - public SnapshotId getSnapshotId() { - return snapshotId; + public Snapshot getSnapshot() { + return snapshot; } /** @@ -124,7 +124,7 @@ public class SnapshotStatus implements ToXContent, Streamable { @Override public void readFrom(StreamInput in) throws IOException { - snapshotId = SnapshotId.readSnapshotId(in); + snapshot = new Snapshot(in); state = State.fromValue(in.readByte()); int size = in.readVInt(); List builder = new ArrayList<>(); @@ -137,7 +137,7 @@ public class SnapshotStatus implements ToXContent, Streamable { @Override public void writeTo(StreamOutput out) throws IOException { - snapshotId.writeTo(out); + snapshot.writeTo(out); out.writeByte(state.value()); out.writeVInt(shards.size()); for (SnapshotIndexShardStatus shard : shards) { @@ -170,7 +170,6 @@ public class SnapshotStatus implements ToXContent, Streamable { } } - /** * Returns number of files in the snapshot */ @@ -178,22 +177,22 @@ public class SnapshotStatus implements ToXContent, Streamable { return stats; } - static final class Fields { - static final String SNAPSHOT = "snapshot"; - static final String REPOSITORY = "repository"; - static final String STATE = "state"; - static final String INDICES = "indices"; - } + private static final String SNAPSHOT = "snapshot"; + private static final String REPOSITORY = "repository"; + private static final String UUID = "uuid"; + private static final String STATE = "state"; + private static final String INDICES = "indices"; @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Fields.SNAPSHOT, snapshotId.getSnapshot()); - builder.field(Fields.REPOSITORY, snapshotId.getRepository()); - builder.field(Fields.STATE, state.name()); + builder.field(SNAPSHOT, snapshot.getSnapshotId().getName()); + builder.field(REPOSITORY, snapshot.getRepository()); + builder.field(UUID, snapshot.getSnapshotId().getUUID()); + builder.field(STATE, state.name()); shardsStats.toXContent(builder, params); stats.toXContent(builder, params); - builder.startObject(Fields.INDICES); + builder.startObject(INDICES); for (SnapshotIndexStatus indexStatus : getIndices().values()) { indexStatus.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index bc139389460..8e7361c1928 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -29,7 +29,7 @@ import org.elasticsearch.action.support.nodes.BaseNodesResponse; import org.elasticsearch.action.support.nodes.TransportNodesAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -94,11 +94,11 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction> snapshotMapBuilder = new HashMap<>(); + Map> snapshotMapBuilder = new HashMap<>(); try { String nodeId = clusterService.localNode().getId(); - for (SnapshotId snapshotId : request.snapshotIds) { - Map shardsStatus = snapshotShardsService.currentSnapshotShards(snapshotId); + for (Snapshot snapshot : request.snapshots) { + Map shardsStatus = snapshotShardsService.currentSnapshotShards(snapshot); if (shardsStatus == null) { continue; } @@ -114,7 +114,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction { - private SnapshotId[] snapshotIds; + private Snapshot[] snapshots; public Request() { } @@ -138,8 +138,8 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction snapshotIds; + private List snapshots; public NodeRequest() { } NodeRequest(String nodeId, TransportNodesSnapshotsStatus.Request request) { super(nodeId); - snapshotIds = Arrays.asList(request.snapshotIds); + snapshots = Arrays.asList(request.snapshots); } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - snapshotIds = in.readList(SnapshotId::readSnapshotId); + snapshots = in.readList(Snapshot::new); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeStreamableList(snapshotIds); + out.writeList(snapshots); } } public static class NodeSnapshotStatus extends BaseNodeResponse { - private Map> status; + private Map> status; NodeSnapshotStatus() { } - public NodeSnapshotStatus(DiscoveryNode node, Map> status) { + public NodeSnapshotStatus(DiscoveryNode node, Map> status) { super(node); this.status = status; } - public Map> status() { + public Map> status() { return status; } @@ -222,9 +222,9 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction> snapshotMapBuilder = new HashMap<>(numberOfSnapshots); + Map> snapshotMapBuilder = new HashMap<>(numberOfSnapshots); for (int i = 0; i < numberOfSnapshots; i++) { - SnapshotId snapshotId = SnapshotId.readSnapshotId(in); + Snapshot snapshot = new Snapshot(in); int numberOfShards = in.readVInt(); Map shardMapBuilder = new HashMap<>(numberOfShards); for (int j = 0; j < numberOfShards; j++) { @@ -232,7 +232,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction> entry : status.entrySet()) { + for (Map.Entry> entry : status.entrySet()) { entry.getKey().writeTo(out); out.writeVInt(entry.getValue().size()); for (Map.Entry shardEntry : entry.getValue().entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index ee60a919da6..afbb2179c0a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -29,26 +29,32 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; /** */ @@ -87,8 +93,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction listener) throws Exception { - List currentSnapshots = snapshotsService.currentSnapshots(request.repository(), request.snapshots()); - + List currentSnapshots = + snapshotsService.currentSnapshots(request.repository(), Arrays.asList(request.snapshots())); if (currentSnapshots.isEmpty()) { listener.onResponse(buildResponse(request, currentSnapshots, null)); return; @@ -105,19 +111,19 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction() { @Override public void onResponse(TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) { try { List currentSnapshots = - snapshotsService.currentSnapshots(request.repository(), request.snapshots()); + snapshotsService.currentSnapshots(request.repository(), Arrays.asList(request.snapshots())); listener.onResponse(buildResponse(request, currentSnapshots, nodeSnapshotStatuses)); } catch (Throwable e) { listener.onFailure(e); @@ -136,12 +142,12 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction currentSnapshots, + private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, List currentSnapshotEntries, TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) throws IOException { // First process snapshot that are currently processed List builder = new ArrayList<>(); - Set currentSnapshotIds = new HashSet<>(); - if (!currentSnapshots.isEmpty()) { + Set currentSnapshotNames = new HashSet<>(); + if (!currentSnapshotEntries.isEmpty()) { Map nodeSnapshotStatusMap; if (nodeSnapshotStatuses != null) { nodeSnapshotStatusMap = nodeSnapshotStatuses.getNodesMap(); @@ -149,8 +155,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction(); } - for (SnapshotsInProgress.Entry entry : currentSnapshots) { - currentSnapshotIds.add(entry.snapshotId()); + for (SnapshotsInProgress.Entry entry : currentSnapshotEntries) { + currentSnapshotNames.add(entry.snapshot().getSnapshotId().getName()); List shardStatusBuilder = new ArrayList<>(); for (ObjectObjectCursor shardEntry : entry.shards()) { SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.value; @@ -158,7 +164,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction shardStatues = nodeStatus.status().get(entry.snapshotId()); + Map shardStatues = nodeStatus.status().get(entry.snapshot()); if (shardStatues != null) { SnapshotIndexShardStatus shardStatus = shardStatues.get(shardEntry.key); if (shardStatus != null) { @@ -190,41 +196,50 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction 0) { - for (String snapshotName : request.snapshots()) { - SnapshotId snapshotId = new SnapshotId(request.repository(), snapshotName); - if (currentSnapshotIds.contains(snapshotId)) { - // This is a snapshot the is currently running - skipping + final String repositoryName = request.repository(); + if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) { + final Set requestedSnapshotNames = Sets.newHashSet(request.snapshots()); + final Map matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream() + .filter(s -> requestedSnapshotNames.contains(s.getName())) + .collect(Collectors.toMap(SnapshotId::getName, Function.identity())); + for (final String snapshotName : request.snapshots()) { + SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName); + if (snapshotId == null) { + if (currentSnapshotNames.contains(snapshotName)) { + // we've already found this snapshot in the current snapshot entries, so skip over continue; + } else { + // neither in the current snapshot entries nor found in the repository + throw new SnapshotMissingException(repositoryName, snapshotName); } - SnapshotInfo snapshot = snapshotsService.snapshot(snapshotId); - List shardStatusBuilder = new ArrayList<>(); - if (snapshot.state().completed()) { - Map shardStatues = snapshotsService.snapshotShards(snapshotId); - for (Map.Entry shardStatus : shardStatues.entrySet()) { - shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue())); - } - final SnapshotsInProgress.State state; - switch (snapshot.state()) { - case FAILED: - state = SnapshotsInProgress.State.FAILED; - break; - case SUCCESS: - case PARTIAL: - // Translating both PARTIAL and SUCCESS to SUCCESS for now - // TODO: add the differentiation on the metadata level in the next major release - state = SnapshotsInProgress.State.SUCCESS; - break; - default: - throw new IllegalArgumentException("Unknown snapshot state " + snapshot.state()); - } - builder.add(new SnapshotStatus(snapshotId, state, Collections.unmodifiableList(shardStatusBuilder))); + } + SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId); + List shardStatusBuilder = new ArrayList<>(); + if (snapshotInfo.state().completed()) { + Map shardStatues = + snapshotsService.snapshotShards(request.repository(), snapshotInfo); + for (Map.Entry shardStatus : shardStatues.entrySet()) { + shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue())); } + final SnapshotsInProgress.State state; + switch (snapshotInfo.state()) { + case FAILED: + state = SnapshotsInProgress.State.FAILED; + break; + case SUCCESS: + case PARTIAL: + // Translating both PARTIAL and SUCCESS to SUCCESS for now + // TODO: add the differentiation on the metadata level in the next major release + state = SnapshotsInProgress.State.SUCCESS; + break; + default: + throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state()); + } + builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotInfo.snapshotId()), state, Collections.unmodifiableList(shardStatusBuilder))); } } } diff --git a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index a083476ea2f..55a09f87f75 100644 --- a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -21,7 +21,7 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.cluster.ClusterState.Custom; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * Meta data about restore processes that are currently executing @@ -73,22 +74,6 @@ public class RestoreInProgress extends AbstractDiffable implements Custo return this.entries; } - /** - * Returns currently running restore process with corresponding snapshot id or null if this snapshot is not being - * restored - * - * @param snapshotId snapshot id - * @return restore metadata or null - */ - public Entry snapshot(SnapshotId snapshotId) { - for (Entry entry : entries) { - if (snapshotId.equals(entry.snapshotId())) { - return entry; - } - } - return null; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -111,22 +96,22 @@ public class RestoreInProgress extends AbstractDiffable implements Custo */ public static class Entry { private final State state; - private final SnapshotId snapshotId; + private final Snapshot snapshot; private final ImmutableOpenMap shards; private final List indices; /** * Creates new restore metadata * - * @param snapshotId snapshot id + * @param snapshot snapshot * @param state current state of the restore process * @param indices list of indices being restored * @param shards map of shards being restored to their current restore status */ - public Entry(SnapshotId snapshotId, State state, List indices, ImmutableOpenMap shards) { - this.snapshotId = snapshotId; - this.state = state; - this.indices = indices; + public Entry(Snapshot snapshot, State state, List indices, ImmutableOpenMap shards) { + this.snapshot = Objects.requireNonNull(snapshot); + this.state = Objects.requireNonNull(state); + this.indices = Objects.requireNonNull(indices); if (shards == null) { this.shards = ImmutableOpenMap.of(); } else { @@ -135,12 +120,12 @@ public class RestoreInProgress extends AbstractDiffable implements Custo } /** - * Returns snapshot id + * Returns snapshot * - * @return snapshot id + * @return snapshot */ - public SnapshotId snapshotId() { - return this.snapshotId; + public Snapshot snapshot() { + return this.snapshot; } /** @@ -172,26 +157,22 @@ public class RestoreInProgress extends AbstractDiffable implements Custo @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Entry entry = (Entry) o; - - if (!indices.equals(entry.indices)) return false; - if (!snapshotId.equals(entry.snapshotId)) return false; - if (!shards.equals(entry.shards)) return false; - if (state != entry.state) return false; - - return true; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + @SuppressWarnings("unchecked") Entry entry = (Entry) o; + return snapshot.equals(entry.snapshot) && + state == entry.state && + indices.equals(entry.indices) && + shards.equals(entry.shards); } @Override public int hashCode() { - int result = state.hashCode(); - result = 31 * result + snapshotId.hashCode(); - result = 31 * result + shards.hashCode(); - result = 31 * result + indices.hashCode(); - return result; + return Objects.hash(snapshot, state, indices, shards); } } @@ -301,31 +282,29 @@ public class RestoreInProgress extends AbstractDiffable implements Custo @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - ShardRestoreStatus status = (ShardRestoreStatus) o; - - if (nodeId != null ? !nodeId.equals(status.nodeId) : status.nodeId != null) return false; - if (reason != null ? !reason.equals(status.reason) : status.reason != null) return false; - if (state != status.state) return false; - - return true; + @SuppressWarnings("unchecked") ShardRestoreStatus status = (ShardRestoreStatus) o; + return state == status.state && + Objects.equals(nodeId, status.nodeId) && + Objects.equals(reason, status.reason); } @Override public int hashCode() { - int result = state != null ? state.hashCode() : 0; - result = 31 * result + (nodeId != null ? nodeId.hashCode() : 0); - result = 31 * result + (reason != null ? reason.hashCode() : 0); - return result; + return Objects.hash(state, nodeId, reason); } } /** * Shard restore process state */ - public static enum State { + public enum State { /** * Initializing state */ @@ -409,7 +388,7 @@ public class RestoreInProgress extends AbstractDiffable implements Custo public RestoreInProgress readFrom(StreamInput in) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { - SnapshotId snapshotId = SnapshotId.readSnapshotId(in); + Snapshot snapshot = new Snapshot(in); State state = State.fromValue(in.readByte()); int indices = in.readVInt(); List indexBuilder = new ArrayList<>(); @@ -423,7 +402,7 @@ public class RestoreInProgress extends AbstractDiffable implements Custo ShardRestoreStatus shardState = ShardRestoreStatus.readShardRestoreStatus(in); builder.put(shardId, shardState); } - entries[i] = new Entry(snapshotId, state, Collections.unmodifiableList(indexBuilder), builder.build()); + entries[i] = new Entry(snapshot, state, Collections.unmodifiableList(indexBuilder), builder.build()); } return new RestoreInProgress(entries); } @@ -435,7 +414,7 @@ public class RestoreInProgress extends AbstractDiffable implements Custo public void writeTo(StreamOutput out) throws IOException { out.writeVInt(entries.size()); for (Entry entry : entries) { - entry.snapshotId().writeTo(out); + entry.snapshot().writeTo(out); out.writeByte(entry.state().value()); out.writeVInt(entry.indices().size()); for (String index : entry.indices()) { @@ -471,8 +450,8 @@ public class RestoreInProgress extends AbstractDiffable implements Custo */ public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); - builder.field("snapshot", entry.snapshotId().getSnapshot()); - builder.field("repository", entry.snapshotId().getRepository()); + builder.field("snapshot", entry.snapshot().getSnapshotId().getName()); + builder.field("repository", entry.snapshot().getRepository()); builder.field("state", entry.state()); builder.startArray("indices"); { diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 5432c1f0f19..f0a0fdec665 100644 --- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -23,13 +23,13 @@ import com.carrotsearch.hppc.ObjectContainer; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.cluster.ClusterState.Custom; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.Snapshot; import java.io.IOException; import java.util.ArrayList; @@ -66,7 +66,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus public static class Entry { private final State state; - private final SnapshotId snapshotId; + private final Snapshot snapshot; private final boolean includeGlobalState; private final boolean partial; private final ImmutableOpenMap shards; @@ -74,9 +74,10 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus private final ImmutableOpenMap> waitingIndices; private final long startTime; - public Entry(SnapshotId snapshotId, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, ImmutableOpenMap shards) { + public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, + ImmutableOpenMap shards) { this.state = state; - this.snapshotId = snapshotId; + this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; this.partial = partial; this.indices = indices; @@ -91,15 +92,15 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus } public Entry(Entry entry, State state, ImmutableOpenMap shards) { - this(entry.snapshotId, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards); + this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards); } public Entry(Entry entry, ImmutableOpenMap shards) { this(entry, entry.state, shards); } - public SnapshotId snapshotId() { - return this.snapshotId; + public Snapshot snapshot() { + return this.snapshot; } public ImmutableOpenMap shards() { @@ -142,7 +143,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus if (startTime != entry.startTime) return false; if (!indices.equals(entry.indices)) return false; if (!shards.equals(entry.shards)) return false; - if (!snapshotId.equals(entry.snapshotId)) return false; + if (!snapshot.equals(entry.snapshot)) return false; if (state != entry.state) return false; if (!waitingIndices.equals(entry.waitingIndices)) return false; @@ -152,7 +153,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus @Override public int hashCode() { int result = state.hashCode(); - result = 31 * result + snapshotId.hashCode(); + result = 31 * result + snapshot.hashCode(); result = 31 * result + (includeGlobalState ? 1 : 0); result = 31 * result + (partial ? 1 : 0); result = 31 * result + shards.hashCode(); @@ -162,6 +163,11 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus return result; } + @Override + public String toString() { + return snapshot.toString(); + } + private ImmutableOpenMap> findWaitingIndices(ImmutableOpenMap shards) { Map> waitingIndicesMap = new HashMap<>(); for (ObjectObjectCursor entry : shards) { @@ -277,7 +283,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus } } - public static enum State { + public enum State { INIT((byte) 0, false, false), STARTED((byte) 1, false, false), SUCCESS((byte) 2, true, false), @@ -347,9 +353,10 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus return this.entries; } - public Entry snapshot(SnapshotId snapshotId) { + public Entry snapshot(final Snapshot snapshot) { for (Entry entry : entries) { - if (snapshotId.equals(entry.snapshotId())) { + final Snapshot curr = entry.snapshot(); + if (curr.equals(snapshot)) { return entry; } } @@ -365,7 +372,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus public SnapshotsInProgress readFrom(StreamInput in) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { - SnapshotId snapshotId = SnapshotId.readSnapshotId(in); + Snapshot snapshot = new Snapshot(in); boolean includeGlobalState = in.readBoolean(); boolean partial = in.readBoolean(); State state = State.fromValue(in.readByte()); @@ -383,7 +390,13 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus State shardState = State.fromValue(in.readByte()); builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState)); } - entries[i] = new Entry(snapshotId, includeGlobalState, partial, state, Collections.unmodifiableList(indexBuilder), startTime, builder.build()); + entries[i] = new Entry(snapshot, + includeGlobalState, + partial, + state, + Collections.unmodifiableList(indexBuilder), + startTime, + builder.build()); } return new SnapshotsInProgress(entries); } @@ -392,7 +405,7 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus public void writeTo(StreamOutput out) throws IOException { out.writeVInt(entries.size()); for (Entry entry : entries) { - entry.snapshotId().writeTo(out); + entry.snapshot().writeTo(out); out.writeBoolean(entry.includeGlobalState()); out.writeBoolean(entry.partial()); out.writeByte(entry.state().value()); @@ -410,25 +423,24 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus } } - static final class Fields { - static final String REPOSITORY = "repository"; - static final String SNAPSHOTS = "snapshots"; - static final String SNAPSHOT = "snapshot"; - static final String INCLUDE_GLOBAL_STATE = "include_global_state"; - static final String PARTIAL = "partial"; - static final String STATE = "state"; - static final String INDICES = "indices"; - static final String START_TIME_MILLIS = "start_time_millis"; - static final String START_TIME = "start_time"; - static final String SHARDS = "shards"; - static final String INDEX = "index"; - static final String SHARD = "shard"; - static final String NODE = "node"; - } + private static final String REPOSITORY = "repository"; + private static final String SNAPSHOTS = "snapshots"; + private static final String SNAPSHOT = "snapshot"; + private static final String UUID = "uuid"; + private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; + private static final String PARTIAL = "partial"; + private static final String STATE = "state"; + private static final String INDICES = "indices"; + private static final String START_TIME_MILLIS = "start_time_millis"; + private static final String START_TIME = "start_time"; + private static final String SHARDS = "shards"; + private static final String INDEX = "index"; + private static final String SHARD = "shard"; + private static final String NODE = "node"; @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray(Fields.SNAPSHOTS); + builder.startArray(SNAPSHOTS); for (Entry entry : entries) { toXContent(entry, builder, params); } @@ -438,30 +450,31 @@ public class SnapshotsInProgress extends AbstractDiffable implements Cus public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); - builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository()); - builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot()); - builder.field(Fields.INCLUDE_GLOBAL_STATE, entry.includeGlobalState()); - builder.field(Fields.PARTIAL, entry.partial()); - builder.field(Fields.STATE, entry.state()); - builder.startArray(Fields.INDICES); + builder.field(REPOSITORY, entry.snapshot().getRepository()); + builder.field(SNAPSHOT, entry.snapshot().getSnapshotId().getName()); + builder.field(UUID, entry.snapshot().getSnapshotId().getUUID()); + builder.field(INCLUDE_GLOBAL_STATE, entry.includeGlobalState()); + builder.field(PARTIAL, entry.partial()); + builder.field(STATE, entry.state()); + builder.startArray(INDICES); { for (String index : entry.indices()) { builder.value(index); } } builder.endArray(); - builder.timeValueField(Fields.START_TIME_MILLIS, Fields.START_TIME, entry.startTime()); - builder.startArray(Fields.SHARDS); + builder.timeValueField(START_TIME_MILLIS, START_TIME, entry.startTime()); + builder.startArray(SHARDS); { for (ObjectObjectCursor shardEntry : entry.shards) { ShardId shardId = shardEntry.key; ShardSnapshotStatus status = shardEntry.value; builder.startObject(); { - builder.field(Fields.INDEX, shardId.getIndex()); - builder.field(Fields.SHARD, shardId.getId()); - builder.field(Fields.STATE, status.state()); - builder.field(Fields.NODE, status.nodeId()); + builder.field(INDEX, shardId.getIndex()); + builder.field(SHARD, shardId.getId()); + builder.field(STATE, status.state()); + builder.field(NODE, status.nodeId()); } builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java b/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java deleted file mode 100644 index 88c60f1f07e..00000000000 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cluster.metadata; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; - -import java.io.IOException; - -/** - * Snapshot ID - repository name + snapshot name - */ -public class SnapshotId implements Streamable { - - private String repository; - - private String snapshot; - - // Caching hash code - private int hashCode; - - private SnapshotId() { - } - - /** - * Constructs new snapshot id - * - * @param repository repository name - * @param snapshot snapshot name - */ - public SnapshotId(String repository, String snapshot) { - this.repository = repository; - this.snapshot = snapshot; - this.hashCode = computeHashCode(); - } - - /** - * Returns repository name - * - * @return repository name - */ - public String getRepository() { - return repository; - } - - /** - * Returns snapshot name - * - * @return snapshot name - */ - public String getSnapshot() { - return snapshot; - } - - @Override - public String toString() { - return repository + ":" + snapshot; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null) return false; - SnapshotId snapshotId = (SnapshotId) o; - return snapshot.equals(snapshotId.snapshot) && repository.equals(snapshotId.repository); - } - - @Override - public int hashCode() { - return hashCode; - } - - private int computeHashCode() { - int result = repository != null ? repository.hashCode() : 0; - result = 31 * result + snapshot.hashCode(); - return result; - } - - /** - * Reads snapshot id from stream input - * - * @param in stream input - * @return snapshot id - */ - public static SnapshotId readSnapshotId(StreamInput in) throws IOException { - SnapshotId snapshot = new SnapshotId(); - snapshot.readFrom(in); - return snapshot; - } - - /** - * {@inheritDoc} - */ - @Override - public void readFrom(StreamInput in) throws IOException { - repository = in.readString(); - snapshot = in.readString(); - hashCode = computeHashCode(); - } - - /** - * {@inheritDoc} - */ - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(repository); - out.writeString(snapshot); - } -} diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 8bb0f51e22e..0fe49369177 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -376,14 +376,20 @@ public class IndexRoutingTable extends AbstractDiffable imple * Initializes a new empty index, to be restored from a snapshot */ public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) { - return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]")); + final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, + "restore_source[" + restoreSource.snapshot().getRepository() + "/" + + restoreSource.snapshot().getSnapshotId().getName() + "]"); + return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, unassignedInfo); } /** * Initializes an existing index, to be restored from a snapshot */ public Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource) { - return initializeAsRestore(indexMetaData, restoreSource, null, false, new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]")); + final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, + "restore_source[" + restoreSource.snapshot().getRepository() + "/" + + restoreSource.snapshot().getSnapshotId().getName() + "]"); + return initializeAsRestore(indexMetaData, restoreSource, null, false, unassignedInfo); } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java index c091f71798b..f80e55ed8b3 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java @@ -20,7 +20,7 @@ package org.elasticsearch.cluster.routing; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -28,13 +28,14 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Objects; /** * Represents snapshot and index from which a recovering index should be restored */ public class RestoreSource implements Streamable, ToXContent { - private SnapshotId snapshotId; + private Snapshot snapshot; private String index; @@ -43,14 +44,14 @@ public class RestoreSource implements Streamable, ToXContent { RestoreSource() { } - public RestoreSource(SnapshotId snapshotId, Version version, String index) { - this.snapshotId = snapshotId; - this.version = version; - this.index = index; + public RestoreSource(Snapshot snapshot, Version version, String index) { + this.snapshot = Objects.requireNonNull(snapshot); + this.version = Objects.requireNonNull(version); + this.index = Objects.requireNonNull(index); } - public SnapshotId snapshotId() { - return snapshotId; + public Snapshot snapshot() { + return snapshot; } public String index() { @@ -61,26 +62,20 @@ public class RestoreSource implements Streamable, ToXContent { return version; } - public static RestoreSource readRestoreSource(StreamInput in) throws IOException { - RestoreSource restoreSource = new RestoreSource(); - restoreSource.readFrom(in); - return restoreSource; - } - public static RestoreSource readOptionalRestoreSource(StreamInput in) throws IOException { return in.readOptionalStreamable(RestoreSource::new); } @Override public void readFrom(StreamInput in) throws IOException { - snapshotId = SnapshotId.readSnapshotId(in); + snapshot = new Snapshot(in); version = Version.readVersion(in); index = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { - snapshotId.writeTo(out); + snapshot.writeTo(out); Version.writeVersion(version, out); out.writeString(index); } @@ -88,8 +83,8 @@ public class RestoreSource implements Streamable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject() - .field("repository", snapshotId.getRepository()) - .field("snapshot", snapshotId.getSnapshot()) + .field("repository", snapshot.getRepository()) + .field("snapshot", snapshot.getSnapshotId().getName()) .field("version", version.toString()) .field("index", index) .endObject(); @@ -97,26 +92,24 @@ public class RestoreSource implements Streamable, ToXContent { @Override public String toString() { - return snapshotId.toString(); + return snapshot.toString(); } @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - RestoreSource that = (RestoreSource) o; - - if (!index.equals(that.index)) return false; - if (!snapshotId.equals(that.snapshotId)) return false; - - return true; + @SuppressWarnings("unchecked") RestoreSource that = (RestoreSource) o; + return snapshot.equals(that.snapshot) && index.equals(that.index); } @Override public int hashCode() { - int result = snapshotId.hashCode(); - result = 31 * result + index.hashCode(); - return result; + return Objects.hash(snapshot, index); } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index adcf36f694f..44445f0b6dd 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RestoreSource; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Booleans; @@ -1442,7 +1441,7 @@ public class IndexShard extends AbstractIndexShardComponent { markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(() -> { try { - final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository()); + final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshot().getRepository()); if (restoreFromRepository(indexShardRepository)) { recoveryListener.onRecoveryDone(recoveryState); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index c7d6f35eff2..62173f936c5 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -395,7 +395,7 @@ final class StoreRecovery { throw new IndexShardRestoreFailedException(shardId, "empty restore source"); } if (logger.isTraceEnabled()) { - logger.trace("[{}] restoring shard [{}]", restoreSource.snapshotId(), shardId); + logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId); } try { translogState.totalOperations(0); @@ -405,7 +405,7 @@ final class StoreRecovery { if (!shardId.getIndexName().equals(restoreSource.index())) { snapshotShardId = new ShardId(restoreSource.index(), IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id()); } - indexShardRepository.restore(restoreSource.snapshotId(), restoreSource.version(), shardId, snapshotShardId, indexShard.recoveryState()); + indexShardRepository.restore(restoreSource.snapshot().getSnapshotId(), restoreSource.version(), shardId, snapshotShardId, indexShard.recoveryState()); indexShard.skipTranslogRecovery(); indexShard.finalizeRecovery(); indexShard.postRecovery("restore done"); diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java index ca481e1430d..5988d82def2 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.snapshots; import org.apache.lucene.index.IndexCommit; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.recovery.RecoveryState; diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index c15d2cfcdbe..e0032fe503b 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -32,7 +32,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseFieldMatcher; @@ -204,7 +204,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements try { snapshotContext.restore(); } catch (Throwable e) { - throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId.getSnapshot() + "]", e); + throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e); } } @@ -318,7 +318,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements int fileListGeneration = tuple.v2(); try { - indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getSnapshot()); + indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getName()); } catch (IOException e) { logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId); } @@ -326,7 +326,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements // Build a list of snapshots that should be preserved List newSnapshotsList = new ArrayList<>(); for (SnapshotFiles point : snapshots) { - if (!point.snapshot().equals(snapshotId.getSnapshot())) { + if (!point.snapshot().equals(snapshotId.getName())) { newSnapshotsList.add(point); } } @@ -339,7 +339,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements */ public BlobStoreIndexShardSnapshot loadSnapshot() { try { - return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getSnapshot()); + return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getName()); } catch (IOException ex) { throw new IndexShardRestoreFailedException(shardId, "failed to read shard snapshot file", ex); } @@ -605,14 +605,14 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements // now create and write the commit point snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FINALIZE); - BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getSnapshot(), + BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), snapshotIndexCommit.getGeneration(), indexCommitPointFiles, snapshotStatus.startTime(), // snapshotStatus.startTime() is assigned on the same machine, so it's safe to use with VLong System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize); //TODO: The time stored in snapshot doesn't include cleanup time. logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId); try { - indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getSnapshot()); + indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getName()); } catch (IOException e) { throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e); } diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index dcf055275e5..719fb812c74 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -645,7 +645,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent * Typical snapshot usage pattern: *
    - *
  • Master calls {@link #initializeSnapshot(org.elasticsearch.cluster.metadata.SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)} + *
  • Master calls {@link #initializeSnapshot(SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)} * with list of indices that will be included into the snapshot
    • *
  • Data nodes call {@link org.elasticsearch.index.snapshots.IndexShardRepository#snapshot(SnapshotId, ShardId, IndexCommit, IndexShardSnapshotStatus)} for each shard
    • *
  • When all shard calls return master calls {@link #finalizeSnapshot} @@ -51,7 +52,7 @@ public interface Repository extends LifecycleComponent { /** * Reads snapshot description from repository. * - * @param snapshotId snapshot ID + * @param snapshotId snapshot id * @return information about snapshot */ SnapshotInfo readSnapshot(SnapshotId snapshotId); @@ -65,10 +66,11 @@ public interface Repository extends LifecycleComponent { * @param indices list of indices * @return information about snapshot */ - MetaData readSnapshotMetaData(SnapshotId snapshotId, SnapshotInfo snapshot, List indices) throws IOException; + MetaData readSnapshotMetaData(SnapshotInfo snapshot, List indices) throws IOException; /** - * Returns the list of snapshots currently stored in the repository + * Returns the list of snapshots currently stored in the repository that match the given predicate on the snapshot name. + * To get all snapshots, the predicate filter should return true regardless of the input. * * @return snapshot list */ diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 121df3e5832..506e64cb9a5 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -24,7 +24,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -34,6 +34,7 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.NotXContentException; import org.elasticsearch.common.io.Streams; @@ -56,7 +57,6 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.RepositoryVerificationException; -import org.elasticsearch.snapshots.InvalidSnapshotNameException; import org.elasticsearch.snapshots.SnapshotCreationException; import org.elasticsearch.snapshots.SnapshotException; import org.elasticsearch.snapshots.SnapshotInfo; @@ -71,6 +71,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * BlobStore - based implementation of Snapshot Repository @@ -130,7 +131,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent s.getName().equals(snapshotName))) { + throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with the same name already exists"); + } + if (snapshotFormat.exists(snapshotsBlobContainer, blobId(snapshotId)) || + snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotName)) { + throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with such name already exists"); } // Write Global MetaData - globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotId.getSnapshot()); + globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotName); for (String index : indices) { final IndexMetaData indexMetaData = metaData.index(index); final BlobPath indexPath = basePath().add("indices").add(index); final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath); - indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getSnapshot()); + indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotName); } } catch (IOException ex) { - throw new SnapshotCreationException(snapshotId, ex); + throw new SnapshotCreationException(repositoryName, snapshotId, ex); } } @@ -314,35 +320,27 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent snapshotIds = snapshots(); - if (snapshotIds.contains(snapshotId)) { - List builder = new ArrayList<>(); - for (SnapshotId id : snapshotIds) { - if (!snapshotId.equals(id)) { - builder.add(id); - } - } - snapshotIds = Collections.unmodifiableList(builder); - } + List snapshotIds = snapshots().stream().filter(id -> snapshotId.equals(id) == false).collect(Collectors.toList()); writeSnapshotList(snapshotIds); // Now delete all indices for (String index : indices) { BlobPath indexPath = basePath().add("indices").add(index); BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath); try { - indexMetaDataFormat(snapshot.version()).delete(indexMetaDataBlobContainer, snapshotId.getSnapshot()); + indexMetaDataFormat(snapshot.version()).delete(indexMetaDataBlobContainer, snapshotId.getName()); } catch (IOException ex) { logger.warn("[{}] failed to delete metadata for index [{}]", ex, snapshotId, index); } @@ -368,10 +366,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent indices, long startTime, String failure, int totalShards, List shardFailures) { + public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, + final List indices, + final long startTime, + final String failure, + final int totalShards, + final List shardFailures) { try { - SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId.getSnapshot(), indices, startTime, failure, System.currentTimeMillis(), totalShards, shardFailures); - snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getSnapshot()); + SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, + indices, + startTime, + failure, + System.currentTimeMillis(), + totalShards, + shardFailures); + snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, blobId(snapshotId)); List snapshotIds = snapshots(); if (!snapshotIds.contains(snapshotId)) { snapshotIds = new ArrayList<>(snapshotIds); @@ -405,15 +414,22 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent legacyPrefixLength) { - name = blobName.substring(prefixLength, blobName.length() - suffixLength); + final String str = blobName.substring(prefixLength, blobName.length() - suffixLength); + // TODO: this will go away once we make the snapshot file writes atomic and + // use it as the source of truth for the snapshots list instead of listing blobs + Tuple pair = parseNameUUIDFromBlobName(str); + name = pair.v1(); + uuid = pair.v2(); } else if (blobName.startsWith(LEGACY_SNAPSHOT_PREFIX) && blobName.length() > suffixLength + prefixLength) { name = blobName.substring(legacyPrefixLength); + uuid = SnapshotId.UNASSIGNED_UUID; } else { // not sure what it was - ignore continue; } - snapshots.add(new SnapshotId(repositoryName, name)); + snapshots.add(new SnapshotId(name, uuid)); } return Collections.unmodifiableList(snapshots); } catch (IOException ex) { @@ -425,28 +441,25 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent indices) throws IOException { - return readSnapshotMetaData(snapshotId, snapshot.version(), indices, false); + public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List indices) throws IOException { + return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false); } - /** - * {@inheritDoc} - */ @Override - public SnapshotInfo readSnapshot(SnapshotId snapshotId) { + public SnapshotInfo readSnapshot(final SnapshotId snapshotId) { try { - return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getSnapshot()); + return snapshotFormat.read(snapshotsBlobContainer, blobId(snapshotId)); } catch (FileNotFoundException | NoSuchFileException ex) { // File is missing - let's try legacy format instead try { - return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getSnapshot()); + return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getName()); } catch (FileNotFoundException | NoSuchFileException ex1) { - throw new SnapshotMissingException(snapshotId, ex); + throw new SnapshotMissingException(repositoryName, snapshotId, ex); } catch (IOException | NotXContentException ex1) { - throw new SnapshotException(snapshotId, "failed to get snapshots", ex1); + throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex1); } } catch (IOException | NotXContentException ex) { - throw new SnapshotException(snapshotId, "failed to get snapshots", ex); + throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex); } } @@ -456,27 +469,27 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent @@ -561,9 +578,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent parseNameUUIDFromBlobName(final String str) { + final String name; + final String uuid; + final int sizeOfUUID = 22; // uuid is 22 chars in length + // unreliable, but highly unlikely to have a snapshot name with a dash followed by 22 characters, + // and this will go away before a release (see #18156). + //norelease + if (str.length() > sizeOfUUID + 1 && str.charAt(str.length() - sizeOfUUID - 1) == '-') { + // new naming convention, snapshot blob id has name and uuid + final int idx = str.length() - sizeOfUUID - 1; + name = str.substring(0, idx); + uuid = str.substring(idx + 1); + } else { + // old naming convention, before snapshots had UUIDs + name = str; + uuid = SnapshotId.UNASSIGNED_UUID; + } + return Tuple.tuple(name, uuid); + } + + // Package private for testing + static String blobId(final SnapshotId snapshotId) { + final String uuid = snapshotId.getUUID(); + if (uuid.equals(SnapshotId.UNASSIGNED_UUID)) { + // the old snapshot blob naming + return snapshotId.getName(); + } + return snapshotId.getName() + "-" + uuid; + } } diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java index 77d4f1cc816..baea7fe28f9 100644 --- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java @@ -19,7 +19,7 @@ package org.elasticsearch.repositories.uri; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.url.URLBlobStore; @@ -42,6 +42,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.Function; +import java.util.function.Predicate; /** * Read-only URL-based implementation of the BlobStoreRepository diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java index 4764462d958..07c2611f2ce 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java @@ -154,8 +154,8 @@ public class RestRecoveryAction extends AbstractCatAction { t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName()); t.addCell(state.getTargetNode().getHostName()); t.addCell(state.getTargetNode().getName()); - t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository()); - t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot()); + t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository()); + t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName()); t.addCell(state.getIndex().totalRecoverFiles()); t.addCell(state.getIndex().recoveredFileCount()); t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java index 0d98dd20278..94d178e4db9 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java @@ -79,7 +79,7 @@ public class RestSnapshotAction extends AbstractCatAction { protected Table getTableWithHeader(RestRequest request) { return new Table() .startHeaders() - .addCell("id", "alias:id,snapshotId;desc:unique snapshot id") + .addCell("id", "alias:id,snapshot;desc:unique snapshot") .addCell("status", "alias:s,status;text-align:right;desc:snapshot name") .addCell("start_epoch", "alias:ste,startEpoch;desc:start time in seconds since 1970-01-01 00:00:00") .addCell("start_time", "alias:sti,startTime;desc:start time in HH:MM:SS") @@ -101,7 +101,7 @@ public class RestSnapshotAction extends AbstractCatAction { for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) { table.startRow(); - table.addCell(snapshotStatus.name()); + table.addCell(snapshotStatus.snapshotId().getName()); table.addCell(snapshotStatus.state()); table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS)); table.addCell(dateFormat.print(snapshotStatus.startTime())); diff --git a/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java b/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java index 4a844fb4907..73762e2d3a7 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.rest.RestStatus; @@ -29,7 +28,12 @@ import java.io.IOException; * Thrown when a user tries to start multiple snapshots at the same time */ public class ConcurrentSnapshotExecutionException extends SnapshotException { - public ConcurrentSnapshotExecutionException(SnapshotId snapshot, String msg) { + + public ConcurrentSnapshotExecutionException(final String repositoryName, final String snapshotName, final String msg) { + super(repositoryName, snapshotName, msg); + } + + public ConcurrentSnapshotExecutionException(final Snapshot snapshot, final String msg) { super(snapshot, msg); } diff --git a/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java b/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java index 48949c11d4c..2acc82add36 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.rest.RestStatus; @@ -30,8 +29,8 @@ import java.io.IOException; */ public class InvalidSnapshotNameException extends SnapshotException { - public InvalidSnapshotNameException(SnapshotId snapshot, String desc) { - super(snapshot, "Invalid snapshot name [" + snapshot.getSnapshot() + "], " + desc); + public InvalidSnapshotNameException(final String repositoryName, final String snapshotName, String desc) { + super(repositoryName, snapshotName, "Invalid snapshot name [" + snapshotName + "], " + desc); } public InvalidSnapshotNameException(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f6e6c4aaed5..44ca6fb972e 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.metadata.RepositoriesMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -85,6 +84,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CopyOnWriteArrayList; @@ -116,7 +117,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property. *

    - * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(SnapshotId, ShardId)}, + * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(Snapshot, ShardId)}, * which updates {@link RestoreInProgress} in cluster state or removes it when all shards are completed. In case of * restore failure a normal recovery fail-over process kicks in. */ @@ -153,8 +154,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis private final MetaDataCreateIndexService createIndexService; - private final ClusterSettings dynamicSettings; - private final MetaDataIndexUpgradeService metaDataIndexUpgradeService; private final CopyOnWriteArrayList> listeners = new CopyOnWriteArrayList<>(); @@ -164,7 +163,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis @Inject public RestoreService(Settings settings, ClusterService clusterService, RepositoriesService repositoriesService, TransportService transportService, - AllocationService allocationService, MetaDataCreateIndexService createIndexService, ClusterSettings dynamicSettings, + AllocationService allocationService, MetaDataCreateIndexService createIndexService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, ClusterSettings clusterSettings) { super(settings); this.clusterService = clusterService; @@ -172,7 +171,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis this.transportService = transportService; this.allocationService = allocationService; this.createIndexService = createIndexService; - this.dynamicSettings = dynamicSettings; this.metaDataIndexUpgradeService = metaDataIndexUpgradeService; transportService.registerRequestHandler(UPDATE_RESTORE_ACTION_NAME, UpdateIndexShardRestoreStatusRequest::new, ThreadPool.Names.SAME, new UpdateRestoreStateRequestHandler()); clusterService.add(this); @@ -188,14 +186,20 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis public void restoreSnapshot(final RestoreRequest request, final ActionListener listener) { try { // Read snapshot info and metadata from the repository - Repository repository = repositoriesService.repository(request.repository()); - final SnapshotId snapshotId = new SnapshotId(request.repository(), request.name()); - final SnapshotInfo snapshot = repository.readSnapshot(snapshotId); - List filteredIndices = SnapshotUtils.filterIndices(snapshot.indices(), request.indices(), request.indicesOptions()); - MetaData metaDataIn = repository.readSnapshotMetaData(snapshotId, snapshot, filteredIndices); + Repository repository = repositoriesService.repository(request.repositoryName); + final Optional matchingSnapshotId = repository.snapshots().stream() + .filter(s -> request.snapshotName.equals(s.getName())).findFirst(); + if (matchingSnapshotId.isPresent() == false) { + throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "snapshot does not exist"); + } + final SnapshotId snapshotId = matchingSnapshotId.get(); + final SnapshotInfo snapshotInfo = repository.readSnapshot(snapshotId); + final Snapshot snapshot = new Snapshot(request.repositoryName, snapshotId); + List filteredIndices = SnapshotUtils.filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions()); + MetaData metaDataIn = repository.readSnapshotMetaData(snapshotInfo, filteredIndices); final MetaData metaData; - if (snapshot.version().before(Version.V_2_0_0_beta1)) { + if (snapshotInfo.version().before(Version.V_2_0_0_beta1)) { // ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing in this snapshot: metaData = MetaData.addDefaultUnitsIfNeeded(logger, metaDataIn); } else { @@ -204,7 +208,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } // Make sure that we can restore from this snapshot - validateSnapshotRestorable(snapshotId, snapshot); + validateSnapshotRestorable(request.repositoryName, snapshotInfo); // Find list of indices that we need to restore final Map renamedIndices = renamedIndices(request, filteredIndices); @@ -220,7 +224,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis // same time RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE); if (restoreInProgress != null && !restoreInProgress.entries().isEmpty()) { - throw new ConcurrentSnapshotExecutionException(snapshotId, "Restore process is already running in this cluster"); + throw new ConcurrentSnapshotExecutionException(snapshot, "Restore process is already running in this cluster"); } // Updating cluster state @@ -236,14 +240,14 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis for (Map.Entry indexEntry : renamedIndices.entrySet()) { String index = indexEntry.getValue(); boolean partial = checkPartial(index); - RestoreSource restoreSource = new RestoreSource(snapshotId, snapshot.version(), index); + RestoreSource restoreSource = new RestoreSource(snapshot, snapshotInfo.version(), index); String renamedIndexName = indexEntry.getKey(); IndexMetaData snapshotIndexMetaData = metaData.index(index); snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings); try { snapshotIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(snapshotIndexMetaData); } catch (Exception ex) { - throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + index + "] because it cannot be upgraded", ex); + throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex); } // Check that the index is closed or doesn't exist IndexMetaData currentIndexMetaData = currentState.metaData().index(renamedIndexName); @@ -309,7 +313,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } shards = shardsBuilder.build(); - RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshotId, RestoreInProgress.State.INIT, Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards); + RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, RestoreInProgress.State.INIT, Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards); builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restoreEntry)); } else { shards = ImmutableOpenMap.of(); @@ -322,28 +326,30 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis if (completed(shards)) { // We don't have any indices to restore - we are done - restoreInfo = new RestoreInfo(request.name(), Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), - shards.size(), shards.size() - failedShards(shards)); + restoreInfo = new RestoreInfo(snapshotId.getName(), + Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), + shards.size(), + shards.size() - failedShards(shards)); } RoutingTable rt = rtBuilder.build(); ClusterState updatedState = builder.metaData(mdBuilder).blocks(blocks).routingTable(rt).build(); RoutingAllocation.Result routingResult = allocationService.reroute( ClusterState.builder(updatedState).routingTable(rt).build(), - "restored snapshot [" + snapshotId + "]"); + "restored snapshot [" + snapshot + "]"); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } private void checkAliasNameConflicts(Map renamedIndices, Set aliases) { for (Map.Entry renamedIndex : renamedIndices.entrySet()) { if (aliases.contains(renamedIndex.getKey())) { - throw new SnapshotRestoreException(snapshotId, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name"); + throw new SnapshotRestoreException(snapshot, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name"); } } } private void populateIgnoredShards(String index, IntSet ignoreShards) { - for (SnapshotShardFailure failure : snapshot.shardFailures()) { + for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) { if (index.equals(failure.index())) { ignoreShards.add(failure.shardId()); } @@ -352,11 +358,11 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis private boolean checkPartial(String index) { // Make sure that index was fully snapshotted - if (failed(snapshot, index)) { + if (failed(snapshotInfo, index)) { if (request.partial()) { return true; } else { - throw new SnapshotRestoreException(snapshotId, "index [" + index + "] wasn't fully snapshotted - cannot restore"); + throw new SnapshotRestoreException(snapshot, "index [" + index + "] wasn't fully snapshotted - cannot restore"); } } else { return false; @@ -367,15 +373,15 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis // Index exist - checking that it's closed if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) { // TODO: Enable restore for open indices - throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + renamedIndex + "] because it's open"); + throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because it's open"); } // Index exist - checking if it's partial restore if (partial) { - throw new SnapshotRestoreException(snapshotId, "cannot restore partial index [" + renamedIndex + "] because such index already exists"); + throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists"); } // Make sure that the number of shards is the same. That's the only thing that we cannot change if (currentIndexMetaData.getNumberOfShards() != snapshotIndexMetaData.getNumberOfShards()) { - throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() + + throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() + "] shard from snapshot with [" + snapshotIndexMetaData.getNumberOfShards() + "] shards"); } } @@ -395,7 +401,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis for (String ignoredSetting : ignoreSettings) { if (!Regex.isSimpleMatchPattern(ignoredSetting)) { if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) { - throw new SnapshotRestoreException(snapshotId, "cannot remove setting [" + ignoredSetting + "] on restore"); + throw new SnapshotRestoreException(snapshot, "cannot remove setting [" + ignoredSetting + "] on restore"); } else { settingsMap.remove(ignoredSetting); } @@ -417,7 +423,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } for(Map.Entry entry : normalizedChangeSettings.getAsMap().entrySet()) { if (UNMODIFIABLE_SETTINGS.contains(entry.getKey())) { - throw new SnapshotRestoreException(snapshotId, "cannot modify setting [" + entry.getKey() + "] on restore"); + throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + entry.getKey() + "] on restore"); } else { settingsMap.put(entry.getKey(), entry.getValue()); } @@ -471,7 +477,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } catch (Throwable e) { - logger.warn("[{}][{}] failed to restore snapshot", e, request.repository(), request.name()); + logger.warn("[{}] failed to restore snapshot", e, request.repositoryName + ":" + request.snapshotName); listener.onFailure(e); } } @@ -480,28 +486,28 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis * This method is used by {@link IndexShard} to notify * {@code RestoreService} about shard restore completion. * - * @param snapshotId snapshot id + * @param snapshot snapshot * @param shardId shard id */ - public void indexShardRestoreCompleted(SnapshotId snapshotId, ShardId shardId) { - logger.trace("[{}] successfully restored shard [{}]", snapshotId, shardId); - UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId, + public void indexShardRestoreCompleted(Snapshot snapshot, ShardId shardId) { + logger.trace("[{}] successfully restored shard [{}]", snapshot, shardId); + UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId, new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.SUCCESS)); - transportService.sendRequest(clusterService.state().nodes().getMasterNode(), - UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); + transportService.sendRequest(clusterService.state().nodes().getMasterNode(), + UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); } public final static class RestoreCompletionResponse { - private final SnapshotId snapshotId; + private final Snapshot snapshot; private final RestoreInfo restoreInfo; - private RestoreCompletionResponse(SnapshotId snapshotId, RestoreInfo restoreInfo) { - this.snapshotId = snapshotId; + private RestoreCompletionResponse(final Snapshot snapshot, final RestoreInfo restoreInfo) { + this.snapshot = snapshot; this.restoreInfo = restoreInfo; } - public SnapshotId getSnapshotId() { - return snapshotId; + public Snapshot getSnapshot() { + return snapshot; } public RestoreInfo getRestoreInfo() { @@ -520,7 +526,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis clusterService.submitStateUpdateTask("update snapshot state", new ClusterStateUpdateTask() { private final List drainedRequests = new ArrayList<>(); - private Map>> batchedRestoreInfo = null; + private Map>> batchedRestoreInfo = null; @Override public ClusterState execute(ClusterState currentState) { @@ -549,8 +555,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis final UpdateIndexShardRestoreStatusRequest updateSnapshotState = drainedRequests.get(i); updateSnapshotState.processed = true; - if (entry.snapshotId().equals(updateSnapshotState.snapshotId())) { - logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshotId(), updateSnapshotState.shardId(), updateSnapshotState.status().state()); + if (entry.snapshot().equals(updateSnapshotState.snapshot())) { + logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state()); if (shardsBuilder == null) { shardsBuilder = ImmutableOpenMap.builder(entry.shards()); } @@ -562,16 +568,19 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis if (shardsBuilder != null) { ImmutableOpenMap shards = shardsBuilder.build(); if (!completed(shards)) { - entries.add(new RestoreInProgress.Entry(entry.snapshotId(), RestoreInProgress.State.STARTED, entry.indices(), shards)); + entries.add(new RestoreInProgress.Entry(entry.snapshot(), RestoreInProgress.State.STARTED, entry.indices(), shards)); } else { - logger.info("restore [{}] is done", entry.snapshotId()); + logger.info("restore [{}] is done", entry.snapshot()); if (batchedRestoreInfo == null) { batchedRestoreInfo = new HashMap<>(); } - assert !batchedRestoreInfo.containsKey(entry.snapshotId()); - batchedRestoreInfo.put(entry.snapshotId(), + assert !batchedRestoreInfo.containsKey(entry.snapshot()); + batchedRestoreInfo.put(entry.snapshot(), new Tuple<>( - new RestoreInfo(entry.snapshotId().getSnapshot(), entry.indices(), shards.size(), shards.size() - failedShards(shards)), + new RestoreInfo(entry.snapshot().getSnapshotId().getName(), + entry.indices(), + shards.size(), + shards.size() - failedShards(shards)), shards)); } } else { @@ -592,15 +601,15 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis @Override public void onFailure(String source, @Nullable Throwable t) { for (UpdateIndexShardRestoreStatusRequest request : drainedRequests) { - logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshotId(), request.shardId(), request.status()); + logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshot(), request.shardId(), request.status()); } } @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { if (batchedRestoreInfo != null) { - for (final Entry>> entry : batchedRestoreInfo.entrySet()) { - final SnapshotId snapshotId = entry.getKey(); + for (final Entry>> entry : batchedRestoreInfo.entrySet()) { + final Snapshot snapshot = entry.getKey(); final RestoreInfo restoreInfo = entry.getValue().v1(); final ImmutableOpenMap shards = entry.getValue().v2(); RoutingTable routingTable = newState.getRoutingTable(); @@ -610,13 +619,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis ShardId shardId = shard.key; ShardRouting shardRouting = findPrimaryShard(routingTable, shardId); if (shardRouting != null && !shardRouting.active()) { - logger.trace("[{}][{}] waiting for the shard to start", snapshotId, shardId); + logger.trace("[{}][{}] waiting for the shard to start", snapshot, shardId); waitForStarted.add(shardId); } } } if (waitForStarted.isEmpty()) { - notifyListeners(snapshotId, restoreInfo); + notifyListeners(snapshot, restoreInfo); } else { clusterService.addLast(new ClusterStateListener() { @Override @@ -629,12 +638,12 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis // Shard disappeared (index deleted) or became active if (shardRouting == null || shardRouting.active()) { iterator.remove(); - logger.trace("[{}][{}] shard disappeared or started - removing", snapshotId, shardId); + logger.trace("[{}][{}] shard disappeared or started - removing", snapshot, shardId); } } } if (waitForStarted.isEmpty()) { - notifyListeners(snapshotId, restoreInfo); + notifyListeners(snapshot, restoreInfo); clusterService.remove(this); } } @@ -655,10 +664,10 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis return null; } - private void notifyListeners(SnapshotId snapshotId, RestoreInfo restoreInfo) { + private void notifyListeners(Snapshot snapshot, RestoreInfo restoreInfo) { for (ActionListener listener : listeners) { try { - listener.onResponse(new RestoreCompletionResponse(snapshotId, restoreInfo)); + listener.onResponse(new RestoreCompletionResponse(snapshot, restoreInfo)); } catch (Throwable e) { logger.warn("failed to update snapshot status for [{}]", e, listener); } @@ -695,7 +704,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } String previousIndex = renamedIndices.put(renamedIndex, index); if (previousIndex != null) { - throw new SnapshotRestoreException(new SnapshotId(request.repository(), request.name()), + throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "indices [" + index + "] and [" + previousIndex + "] are renamed into the same index [" + renamedIndex + "]"); } } @@ -705,16 +714,18 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis /** * Checks that snapshots can be restored and have compatible version * - * @param snapshotId snapshot id - * @param snapshot snapshot metadata + * @param repository repository name + * @param snapshotInfo snapshot metadata */ - private void validateSnapshotRestorable(SnapshotId snapshotId, SnapshotInfo snapshot) { - if (!snapshot.state().restorable()) { - throw new SnapshotRestoreException(snapshotId, "unsupported snapshot state [" + snapshot.state() + "]"); + private void validateSnapshotRestorable(final String repository, final SnapshotInfo snapshotInfo) { + if (!snapshotInfo.state().restorable()) { + throw new SnapshotRestoreException(new Snapshot(repository, snapshotInfo.snapshotId()), + "unsupported snapshot state [" + snapshotInfo.state() + "]"); } - if (Version.CURRENT.before(snapshot.version())) { - throw new SnapshotRestoreException(snapshotId, "the snapshot was created with Elasticsearch version [" + - snapshot.version() + "] which is higher than the version of this node [" + Version.CURRENT + "]"); + if (Version.CURRENT.before(snapshotInfo.version())) { + throw new SnapshotRestoreException(new Snapshot(repository, snapshotInfo.snapshotId()), + "the snapshot was created with Elasticsearch version [" + snapshotInfo.version() + + "] which is higher than the version of this node [" + Version.CURRENT + "]"); } } @@ -746,8 +757,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } if (shardsToFail != null) { for (ShardId shardId : shardsToFail) { - logger.trace("[{}] failing running shard restore [{}]", entry.snapshotId(), shardId); - updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshotId(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted"))); + logger.trace("[{}] failing running shard restore [{}]", entry.snapshot(), shardId); + updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshot(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted"))); } } } @@ -757,12 +768,12 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis /** * Fails the given snapshot restore operation for the given shard */ - public void failRestore(SnapshotId snapshotId, ShardId shardId) { - logger.debug("[{}] failed to restore shard [{}]", snapshotId, shardId); - UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId, + public void failRestore(Snapshot snapshot, ShardId shardId) { + logger.debug("[{}] failed to restore shard [{}]", snapshot, shardId); + UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId, new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.FAILURE)); - transportService.sendRequest(clusterService.state().nodes().getMasterNode(), - UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); + transportService.sendRequest(clusterService.state().nodes().getMasterNode(), + UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); } private boolean failed(SnapshotInfo snapshot, String index) { @@ -846,7 +857,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis RestoreInProgress snapshots = clusterState.custom(RestoreInProgress.TYPE); if (snapshots != null) { for (RestoreInProgress.Entry snapshot : snapshots.entries()) { - if (repository.equals(snapshot.snapshotId().getRepository())) { + if (repository.equals(snapshot.snapshot().getRepository())) { return true; } } @@ -861,9 +872,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis final private String cause; - final private String name; + final private String repositoryName; - final private String repository; + final private String snapshotName; final private String[] indices; @@ -890,9 +901,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis /** * Constructs new restore request * - * @param cause cause for restoring the snapshot - * @param repository repository name - * @param name snapshot name + * @param repositoryName repositoryName + * @param snapshotName snapshotName * @param indices list of indices to restore * @param indicesOptions indices options * @param renamePattern pattern to rename indices @@ -903,14 +913,14 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis * @param partial allow partial restore * @param indexSettings index settings that should be changed on restore * @param ignoreIndexSettings index settings that shouldn't be restored + * @param cause cause for restoring the snapshot */ - public RestoreRequest(String cause, String repository, String name, String[] indices, IndicesOptions indicesOptions, + public RestoreRequest(String repositoryName, String snapshotName, String[] indices, IndicesOptions indicesOptions, String renamePattern, String renameReplacement, Settings settings, TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases, - Settings indexSettings, String[] ignoreIndexSettings ) { - this.cause = cause; - this.name = name; - this.repository = repository; + Settings indexSettings, String[] ignoreIndexSettings, String cause) { + this.repositoryName = Objects.requireNonNull(repositoryName); + this.snapshotName = Objects.requireNonNull(snapshotName); this.indices = indices; this.renamePattern = renamePattern; this.renameReplacement = renameReplacement; @@ -922,7 +932,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis this.includeAliases = includeAliases; this.indexSettings = indexSettings; this.ignoreIndexSettings = ignoreIndexSettings; - + this.cause = cause; } /** @@ -934,22 +944,22 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis return cause; } - /** - * Returns snapshot name - * - * @return snapshot name - */ - public String name() { - return name; - } - /** * Returns repository name * * @return repository name */ - public String repository() { - return repository; + public String repositoryName() { + return repositoryName; + } + + /** + * Returns snapshot name + * + * @return snapshot name + */ + public String snapshotName() { + return snapshotName; } /** @@ -1058,7 +1068,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis * Internal class that is used to send notifications about finished shard restore operations to master node */ public static class UpdateIndexShardRestoreStatusRequest extends TransportRequest { - private SnapshotId snapshotId; + private Snapshot snapshot; private ShardId shardId; private ShardRestoreStatus status; @@ -1068,8 +1078,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis } - private UpdateIndexShardRestoreStatusRequest(SnapshotId snapshotId, ShardId shardId, ShardRestoreStatus status) { - this.snapshotId = snapshotId; + private UpdateIndexShardRestoreStatusRequest(Snapshot snapshot, ShardId shardId, ShardRestoreStatus status) { + this.snapshot = snapshot; this.shardId = shardId; this.status = status; } @@ -1077,7 +1087,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - snapshotId = SnapshotId.readSnapshotId(in); + snapshot = new Snapshot(in); shardId = ShardId.readShardId(in); status = ShardRestoreStatus.readShardRestoreStatus(in); } @@ -1085,13 +1095,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - snapshotId.writeTo(out); + snapshot.writeTo(out); shardId.writeTo(out); status.writeTo(out); } - public SnapshotId snapshotId() { - return snapshotId; + public Snapshot snapshot() { + return snapshot; } public ShardId shardId() { @@ -1104,7 +1114,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis @Override public String toString() { - return "" + snapshotId + ", shardId [" + shardId + "], status [" + status.state() + "]"; + return "" + snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]"; } } diff --git a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java new file mode 100644 index 00000000000..314cd4053dd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * Basic information about a snapshot - a SnapshotId and the repository that the snapshot belongs to. + */ +public final class Snapshot implements Writeable { + + private final String repository; + private final SnapshotId snapshotId; + private final int hashCode; + + /** + * Constructs a snapshot. + */ + public Snapshot(final String repository, final SnapshotId snapshotId) { + this.repository = Objects.requireNonNull(repository); + this.snapshotId = Objects.requireNonNull(snapshotId); + this.hashCode = computeHashCode(); + } + + /** + * Constructs a snapshot from the stream input. + */ + public Snapshot(final StreamInput in) throws IOException { + repository = in.readString(); + snapshotId = new SnapshotId(in); + hashCode = computeHashCode(); + } + + /** + * Gets the repository name for the snapshot. + */ + public String getRepository() { + return repository; + } + + /** + * Gets the snapshot id for the snapshot. + */ + public SnapshotId getSnapshotId() { + return snapshotId; + } + + @Override + public String toString() { + return repository + ":" + snapshotId.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + @SuppressWarnings("unchecked") Snapshot that = (Snapshot) o; + return repository.equals(that.repository) && snapshotId.equals(that.snapshotId); + } + + @Override + public int hashCode() { + return hashCode; + } + + private int computeHashCode() { + return Objects.hash(repository, snapshotId); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(repository); + snapshotId.writeTo(out); + } + +} diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java index 58faecda4a4..32d3992f243 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; @@ -29,11 +28,19 @@ import java.io.IOException; */ public class SnapshotCreationException extends SnapshotException { + public SnapshotCreationException(final String repositoryName, final String snapshotName, final Throwable cause) { + super(repositoryName, snapshotName, "failed to create snapshot", cause); + } + + public SnapshotCreationException(final String repositoryName, final SnapshotId snapshotId, final Throwable cause) { + super(repositoryName, snapshotId, "failed to create snapshot", cause); + } + + public SnapshotCreationException(final String repositoryName, final SnapshotId snapshotId, final String msg) { + super(repositoryName, snapshotId, msg); + } + public SnapshotCreationException(StreamInput in) throws IOException { super(in); } - - public SnapshotCreationException(SnapshotId snapshot, Throwable cause) { - super(snapshot, "failed to create snapshot", cause); - } } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java index b109c46f46b..0acd73d62ea 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java @@ -20,7 +20,7 @@ package org.elasticsearch.snapshots; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,33 +30,68 @@ import java.io.IOException; * Generic snapshot exception */ public class SnapshotException extends ElasticsearchException { - private final SnapshotId snapshot; - public SnapshotException(SnapshotId snapshot, String msg) { + @Nullable + private final String repositoryName; + @Nullable + private final String snapshotName; + + public SnapshotException(final Snapshot snapshot, final String msg) { this(snapshot, msg, null); } - public SnapshotException(SnapshotId snapshot, String msg, Throwable cause) { + public SnapshotException(final Snapshot snapshot, final String msg, final Throwable cause) { super("[" + (snapshot == null ? "_na" : snapshot) + "] " + msg, cause); - this.snapshot = snapshot; + if (snapshot != null) { + this.repositoryName = snapshot.getRepository(); + this.snapshotName = snapshot.getSnapshotId().getName(); + } else { + this.repositoryName = null; + this.snapshotName = null; + } } - public SnapshotException(StreamInput in) throws IOException { + public SnapshotException(final String repositoryName, final SnapshotId snapshotId, final String msg) { + this(repositoryName, snapshotId, msg, null); + } + + public SnapshotException(final String repositoryName, final SnapshotId snapshotId, final String msg, final Throwable cause) { + super("[" + repositoryName + ":" + snapshotId + "] " + msg, cause); + this.repositoryName = repositoryName; + this.snapshotName = snapshotId.getName(); + } + + public SnapshotException(final String repositoryName, final String snapshotName, final String msg) { + this(repositoryName, snapshotName, msg, null); + } + + public SnapshotException(final String repositoryName, final String snapshotName, final String msg, final Throwable cause) { + super("[" + repositoryName + ":" + snapshotName + "]" + msg, cause); + this.repositoryName = repositoryName; + this.snapshotName = snapshotName; + } + + public SnapshotException(final StreamInput in) throws IOException { super(in); - if (in.readBoolean()) { - snapshot = SnapshotId.readSnapshotId(in); - } else { - snapshot = null; - } + repositoryName = in.readOptionalString(); + snapshotName = in.readOptionalString(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalStreamable(snapshot); + out.writeOptionalString(repositoryName); + out.writeOptionalString(snapshotName); } - public SnapshotId snapshot() { - return snapshot; + @Nullable + public String getRepositoryName() { + return repositoryName; } + + @Nullable + public String getSnapshotName() { + return snapshotName; + } + } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java new file mode 100644 index 00000000000..16f371b28f7 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * SnapshotId - snapshot name + snapshot UUID + */ +public final class SnapshotId implements Writeable { + + /** + * This value is for older snapshots that don't have a UUID. + */ + public static final String UNASSIGNED_UUID = "_na_"; + + private final String name; + private final String uuid; + + // Caching hash code + private final int hashCode; + + /** + * Constructs a new snapshot + * + * @param name snapshot name + * @param uuid snapshot uuid + */ + public SnapshotId(final String name, final String uuid) { + this.name = Objects.requireNonNull(name); + this.uuid = Objects.requireNonNull(uuid); + this.hashCode = computeHashCode(); + } + + /** + * Constructs a new snapshot from a input stream + * + * @param in input stream + */ + public SnapshotId(final StreamInput in) throws IOException { + name = in.readString(); + uuid = in.readString(); + hashCode = computeHashCode(); + } + + /** + * Returns snapshot name + * + * @return snapshot name + */ + public String getName() { + return name; + } + + /** + * Returns the snapshot UUID + * + * @return snapshot uuid + */ + public String getUUID() { + return uuid; + } + + @Override + public String toString() { + return name + "/" + uuid; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + @SuppressWarnings("unchecked") final SnapshotId that = (SnapshotId) o; + return name.equals(that.name) && uuid.equals(that.uuid); + } + + @Override + public int hashCode() { + return hashCode; + } + + private int computeHashCode() { + return Objects.hash(name, uuid); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(uuid); + } + +} diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 2b8ea8ace31..2159fda2237 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -37,17 +37,19 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * Information about a snapshot */ public final class SnapshotInfo implements Comparable, ToXContent, FromXContentBuilder, Writeable { - public static final SnapshotInfo PROTO = new SnapshotInfo("", Collections.emptyList(), 0); + public static final SnapshotInfo PROTO = new SnapshotInfo(new SnapshotId("", ""), Collections.emptyList(), 0); public static final String CONTEXT_MODE_PARAM = "context_mode"; public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT"; private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime"); private static final String SNAPSHOT = "snapshot"; + private static final String UUID = "uuid"; private static final String INDICES = "indices"; private static final String STATE = "state"; private static final String REASON = "reason"; @@ -68,7 +70,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String TOTAL_SHARDS = "total_shards"; private static final String SUCCESSFUL_SHARDS = "successful_shards"; - private final String name; + private final SnapshotId snapshotId; private final SnapshotState state; @@ -88,39 +90,35 @@ public final class SnapshotInfo implements Comparable, ToXContent, private final List shardFailures; - public SnapshotInfo(String name, List indices, long startTime) { - this(name, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList()); + public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime) { + this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList()); } - public SnapshotInfo(String name, List indices, long startTime, String reason, long endTime, + public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, String reason, long endTime, int totalShards, List shardFailures) { - this(name, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT, + this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT, startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures); } - private SnapshotInfo(String name, List indices, SnapshotState state, String reason, Version version, long startTime, - long endTime, int totalShards, int successfulShards, List shardFailures) { - assert name != null; - assert indices != null; - assert state != null; - assert shardFailures != null; - this.name = name; - this.indices = indices; - this.state = state; + private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, String reason, Version version, + long startTime, long endTime, int totalShards, int successfulShards, List shardFailures) { + this.snapshotId = Objects.requireNonNull(snapshotId); + this.indices = Objects.requireNonNull(indices); + this.state = Objects.requireNonNull(state); this.reason = reason; this.version = version; this.startTime = startTime; this.endTime = endTime; this.totalShards = totalShards; this.successfulShards = successfulShards; - this.shardFailures = shardFailures; + this.shardFailures = Objects.requireNonNull(shardFailures); } /** * Constructs snapshot information from stream input */ public SnapshotInfo(final StreamInput in) throws IOException { - name = in.readString(); + snapshotId = new SnapshotId(in); int size = in.readVInt(); List indicesListBuilder = new ArrayList<>(); for (int i = 0; i < size; i++) { @@ -147,12 +145,12 @@ public final class SnapshotInfo implements Comparable, ToXContent, } /** - * Returns snapshot name + * Returns snapshot id * - * @return snapshot name + * @return snapshot id */ - public String name() { - return name; + public SnapshotId snapshotId() { + return snapshotId; } /** @@ -270,16 +268,21 @@ public final class SnapshotInfo implements Comparable, ToXContent, } final SnapshotInfo that = (SnapshotInfo) o; - return startTime == that.startTime && name.equals(that.name); + return startTime == that.startTime && snapshotId.equals(that.snapshotId); } @Override public int hashCode() { - int result = name.hashCode(); + int result = snapshotId.hashCode(); result = 31 * result + Long.hashCode(startTime); return result; } + @Override + public String toString() { + return "SnapshotInfo[snapshotId=" + snapshotId + ", state=" + state + ", indices=" + indices + "]"; + } + /** * Returns snapshot REST status */ @@ -303,7 +306,8 @@ public final class SnapshotInfo implements Comparable, ToXContent, // write snapshot info for the API and any other situations builder.startObject(); - builder.field(SNAPSHOT, name); + builder.field(SNAPSHOT, snapshotId.getName()); + builder.field(UUID, snapshotId.getUUID()); builder.field(VERSION_ID, version.id); builder.field(VERSION, version.toString()); builder.startArray(INDICES); @@ -342,7 +346,8 @@ public final class SnapshotInfo implements Comparable, ToXContent, private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final ToXContent.Params params) throws IOException { builder.startObject(SNAPSHOT); - builder.field(NAME, name); + builder.field(NAME, snapshotId.getName()); + builder.field(UUID, snapshotId.getUUID()); builder.field(VERSION_ID, version.id); builder.startArray(INDICES); for (String index : indices) { @@ -380,13 +385,14 @@ public final class SnapshotInfo implements Comparable, ToXContent, */ public static SnapshotInfo fromXContent(final XContentParser parser) throws IOException { String name = null; + String uuid = null; Version version = Version.CURRENT; SnapshotState state = SnapshotState.IN_PROGRESS; String reason = null; List indices = Collections.emptyList(); long startTime = 0; long endTime = 0; - int totalShard = 0; + int totalShards = 0; int successfulShards = 0; List shardFailures = Collections.emptyList(); if (parser.currentToken() == null) { // fresh parser? move to the first token @@ -406,6 +412,8 @@ public final class SnapshotInfo implements Comparable, ToXContent, if (token.isValue()) { if (NAME.equals(currentFieldName)) { name = parser.text(); + } else if (UUID.equals(currentFieldName)) { + uuid = parser.text(); } else if (STATE.equals(currentFieldName)) { state = SnapshotState.valueOf(parser.text()); } else if (REASON.equals(currentFieldName)) { @@ -415,7 +423,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, } else if (END_TIME.equals(currentFieldName)) { endTime = parser.longValue(); } else if (TOTAL_SHARDS.equals(currentFieldName)) { - totalShard = parser.intValue(); + totalShards = parser.intValue(); } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) { successfulShards = parser.intValue(); } else if (VERSION_ID.equals(currentFieldName)) { @@ -448,12 +456,25 @@ public final class SnapshotInfo implements Comparable, ToXContent, } else { throw new ElasticsearchParseException("unexpected token [" + token + "]"); } - return new SnapshotInfo(name, indices, state, reason, version, startTime, endTime, totalShard, successfulShards, shardFailures); + if (uuid == null) { + // the old format where there wasn't a UUID + uuid = SnapshotId.UNASSIGNED_UUID; + } + return new SnapshotInfo(new SnapshotId(name, uuid), + indices, + state, + reason, + version, + startTime, + endTime, + totalShards, + successfulShards, + shardFailures); } @Override public void writeTo(final StreamOutput out) throws IOException { - out.writeString(name); + snapshotId.writeTo(out); out.writeVInt(indices.size()); for (String index : indices) { out.writeString(index); diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java index 27fe3de51e0..5f0979e38d8 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.rest.RestStatus; @@ -30,12 +29,16 @@ import java.io.IOException; */ public class SnapshotMissingException extends SnapshotException { - public SnapshotMissingException(SnapshotId snapshot, Throwable cause) { - super(snapshot, "is missing", cause); + public SnapshotMissingException(final String repositoryName, final SnapshotId snapshotId, final Throwable cause) { + super(repositoryName, snapshotId, "is missing", cause); } - public SnapshotMissingException(SnapshotId snapshot) { - super(snapshot, "is missing"); + public SnapshotMissingException(final String repositoryName, final SnapshotId snapshotId) { + super(repositoryName, snapshotId, "is missing"); + } + + public SnapshotMissingException(final String repositoryName, final String snapshotName) { + super(repositoryName, snapshotName, "is missing"); } public SnapshotMissingException(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java index 940f8162e6b..9003a08a54a 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; @@ -28,11 +27,19 @@ import java.io.IOException; * Snapshot restore exception */ public class SnapshotRestoreException extends SnapshotException { - public SnapshotRestoreException(SnapshotId snapshot, String message) { + public SnapshotRestoreException(final String repositoryName, final String snapshotName, final String message) { + super(repositoryName, snapshotName, message); + } + + public SnapshotRestoreException(final String repositoryName, final String snapshotName, final String message, final Throwable cause) { + super(repositoryName, snapshotName, message, cause); + } + + public SnapshotRestoreException(final Snapshot snapshot, final String message) { super(snapshot, message); } - public SnapshotRestoreException(SnapshotId snapshot, String message, Throwable cause) { + public SnapshotRestoreException(final Snapshot snapshot, final String message, final Throwable cause) { super(snapshot, message, cause); } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index c2a33680efa..b9d8c729223 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -94,7 +93,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent shardSnapshots = emptyMap(); + private volatile Map shardSnapshots = emptyMap(); private final BlockingQueue updatedSnapshotStateQueue = ConcurrentCollections.newBlockingQueue(); @@ -176,11 +175,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent * - * @param snapshotId snapshot id + * @param snapshot snapshot * @return map of shard id to snapshot status */ - public Map currentSnapshotShards(SnapshotId snapshotId) { - SnapshotShards snapshotShards = shardSnapshots.get(snapshotId); + public Map currentSnapshotShards(Snapshot snapshot) { + SnapshotShards snapshotShards = shardSnapshots.get(snapshot); if (snapshotShards == null) { return null; } else { @@ -195,24 +194,25 @@ public class SnapshotShardsService extends AbstractLifecycleComponent survivors = new HashMap<>(); + Map survivors = new HashMap<>(); // First, remove snapshots that are no longer there - for (Map.Entry entry : shardSnapshots.entrySet()) { - if (snapshotsInProgress != null && snapshotsInProgress.snapshot(entry.getKey()) != null) { + for (Map.Entry entry : shardSnapshots.entrySet()) { + final Snapshot snapshot = entry.getKey(); + if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) { survivors.put(entry.getKey(), entry.getValue()); } } // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running // snapshots in the future - Map> newSnapshots = new HashMap<>(); + Map> newSnapshots = new HashMap<>(); // Now go through all snapshots and update existing or create missing final String localNodeId = clusterService.localNode().getId(); if (snapshotsInProgress != null) { for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { if (entry.state() == SnapshotsInProgress.State.STARTED) { Map startedShards = new HashMap<>(); - SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshotId()); + SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot()); for (ObjectObjectCursor shard : entry.shards()) { // Add all new shards to start processing on if (localNodeId.equals(shard.value.nodeId())) { @@ -223,7 +223,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent shards = new HashMap<>(); @@ -231,15 +231,15 @@ public class SnapshotShardsService extends AbstractLifecycleComponent shard : entry.shards()) { IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key); @@ -250,16 +250,16 @@ public class SnapshotShardsService extends AbstractLifecycleComponent> entry : newSnapshots.entrySet()) { + for (final Map.Entry> entry : newSnapshots.entrySet()) { for (final Map.Entry shardEntry : entry.getValue().entrySet()) { final ShardId shardId = shardEntry.getKey(); try { @@ -318,11 +318,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent localShards = currentSnapshotShards(snapshot.snapshotId()); + Map localShards = currentSnapshotShards(snapshot.snapshot()); if (localShards != null) { ImmutableOpenMap masterShards = snapshot.shards(); for(Map.Entry localShard : localShards.entrySet()) { @@ -380,13 +380,13 @@ public class SnapshotShardsService extends AbstractLifecycleComponentOnce shard snapshot is created data node updates state of the shard in the cluster state using the {@link SnapshotShardsService#updateIndexShardSnapshotStatus} method

    • *
  • When last shard is completed master node in {@link SnapshotShardsService#innerUpdateSnapshotState} method marks the snapshot as completed
    • *
  • After cluster state is updated, the {@link #endSnapshot(SnapshotsInProgress.Entry)} finalizes snapshot in the repository, - * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(SnapshotId, SnapshotInfo, Throwable)} to remove snapshot from cluster state
    • + * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Throwable)} to remove snapshot from cluster state *
*/ public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateListener { @@ -121,49 +124,67 @@ public class SnapshotsService extends AbstractLifecycleComponent snapshotIds(final String repositoryName) { + Repository repository = repositoriesService.repository(repositoryName); + assert repository != null; // should only be called once we've validated the repository exists + return repository.snapshots(); + } + /** * Retrieves snapshot from repository * - * @param snapshotId snapshot id + * @param repositoryName repository name + * @param snapshotId snapshot id * @return snapshot * @throws SnapshotMissingException if snapshot is not found */ - public SnapshotInfo snapshot(SnapshotId snapshotId) { - validate(snapshotId); - List entries = currentSnapshots(snapshotId.getRepository(), new String[]{snapshotId.getSnapshot()}); + public SnapshotInfo snapshot(final String repositoryName, final SnapshotId snapshotId) { + List entries = currentSnapshots(repositoryName, Arrays.asList(snapshotId.getName())); if (!entries.isEmpty()) { return inProgressSnapshot(entries.iterator().next()); } - return repositoriesService.repository(snapshotId.getRepository()).readSnapshot(snapshotId); + return repositoriesService.repository(repositoryName).readSnapshot(snapshotId); } /** * Returns a list of snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name + * @param snapshotIds snapshots for which to fetch snapshot information + * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning, + * if false, they will throw an error * @return list of snapshots */ - public List snapshots(String repositoryName, boolean ignoreUnavailable) { - Set snapshotSet = new HashSet<>(); - List entries = currentSnapshots(repositoryName, null); + public List snapshots(final String repositoryName, List snapshotIds, final boolean ignoreUnavailable) { + final Set snapshotSet = new HashSet<>(); + final Set snapshotIdsToIterate = new HashSet<>(snapshotIds); + // first, look at the snapshots in progress + final List entries = + currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList())); for (SnapshotsInProgress.Entry entry : entries) { snapshotSet.add(inProgressSnapshot(entry)); + snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId()); } - Repository repository = repositoriesService.repository(repositoryName); - List snapshotIds = repository.snapshots(); - for (SnapshotId snapshotId : snapshotIds) { + // then, look in the repository + final Repository repository = repositoriesService.repository(repositoryName); + for (SnapshotId snapshotId : snapshotIdsToIterate) { try { snapshotSet.add(repository.readSnapshot(snapshotId)); } catch (Exception ex) { if (ignoreUnavailable) { logger.warn("failed to get snapshot [{}]", ex, snapshotId); } else { - throw new SnapshotException(snapshotId, "Snapshot could not be read", ex); + throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex); } } } - - ArrayList snapshotList = new ArrayList<>(snapshotSet); + final ArrayList snapshotList = new ArrayList<>(snapshotSet); CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); } @@ -174,9 +195,9 @@ public class SnapshotsService extends AbstractLifecycleComponent currentSnapshots(String repositoryName) { + public List currentSnapshots(final String repositoryName) { List snapshotList = new ArrayList<>(); - List entries = currentSnapshots(repositoryName, null); + List entries = currentSnapshots(repositoryName, Collections.emptyList()); for (SnapshotsInProgress.Entry entry : entries) { snapshotList.add(inProgressSnapshot(entry)); } @@ -194,8 +215,11 @@ public class SnapshotsService extends AbstractLifecycleComponent indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request.indicesOptions(), request.indices())); - logger.trace("[{}][{}] creating snapshot for indices [{}]", request.repository(), request.name(), indices); - newSnapshot = new SnapshotsInProgress.Entry(snapshotId, request.includeGlobalState(), request.partial(), State.INIT, indices, System.currentTimeMillis(), null); + logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices); + newSnapshot = new SnapshotsInProgress.Entry(new Snapshot(repositoryName, snapshotId), + request.includeGlobalState(), + request.partial(), + State.INIT, + indices, + System.currentTimeMillis(), + null); snapshots = new SnapshotsInProgress(newSnapshot); } else { // TODO: What should we do if a snapshot is already running? - throw new ConcurrentSnapshotExecutionException(snapshotId, "a snapshot is already running"); + throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "a snapshot is already running"); } return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build(); } @Override public void onFailure(String source, Throwable t) { - logger.warn("[{}][{}] failed to create snapshot", t, request.repository(), request.name()); + logger.warn("[{}][{}] failed to create snapshot", t, repositoryName, snapshotName); newSnapshot = null; listener.onFailure(t); } @@ -228,12 +258,9 @@ public class SnapshotsService extends AbstractLifecycleComponent + beginSnapshot(newState, newSnapshot, request.partial(), listener) + ); } } @@ -253,34 +280,36 @@ public class SnapshotsService extends AbstractLifecycleComponent entries = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshots.entries()) { - if (entry.snapshotId().equals(snapshot.snapshotId())) { + if (entry.snapshot().equals(snapshot.snapshot())) { // Replace the snapshot that was just created ImmutableOpenMap shards = shards(currentState, entry.indices()); if (!partial) { @@ -362,13 +394,15 @@ public class SnapshotsService extends AbstractLifecycleComponentemptyList()); + repositoriesService.repository(snapshot.snapshot().getRepository()) + .finalizeSnapshot(snapshot.snapshot().getSnapshotId(), + snapshot.indices(), + snapshot.startTime(), + ExceptionsHelper.detailedMessage(t), + 0, + Collections.emptyList()); } catch (Throwable t2) { - logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshotId()); + logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshot()); } } userCreateSnapshotListener.onFailure(t); @@ -433,7 +472,7 @@ public class SnapshotsService extends AbstractLifecycleComponent * * @param repository repository id - * @param snapshots optional list of snapshots that will be used as a filter + * @param snapshots list of snapshots that will be used as a filter, empty list means no snapshots are filtered * @return list of metadata for currently running snapshots */ - public List currentSnapshots(String repository, String[] snapshots) { + public List currentSnapshots(final String repository, final List snapshots) { SnapshotsInProgress snapshotsInProgress = clusterService.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) { return Collections.emptyList(); @@ -458,12 +497,12 @@ public class SnapshotsService extends AbstractLifecycleComponent 0) { + if (snapshots.isEmpty() == false) { for (String snapshot : snapshots) { - if (entry.snapshotId().getSnapshot().equals(snapshot)) { + if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) { return snapshotsInProgress.entries(); } } @@ -474,12 +513,12 @@ public class SnapshotsService extends AbstractLifecycleComponent builder = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { - if (!entry.snapshotId().getRepository().equals(repository)) { + if (entry.snapshot().getRepository().equals(repository) == false) { continue; } - if (snapshots != null && snapshots.length > 0) { + if (snapshots.isEmpty() == false) { for (String snapshot : snapshots) { - if (entry.snapshotId().getSnapshot().equals(snapshot)) { + if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) { builder.add(entry); break; } @@ -494,34 +533,35 @@ public class SnapshotsService extends AbstractLifecycleComponent - * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(SnapshotId)} because it + * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it * returns similar information but for already finished snapshots. *

* - * @param snapshotId snapshot id + * @param repositoryName repository name + * @param snapshotInfo snapshot info * @return map of shard id to snapshot status */ - public Map snapshotShards(SnapshotId snapshotId) throws IOException { - validate(snapshotId); + public Map snapshotShards(final String repositoryName, + final SnapshotInfo snapshotInfo) throws IOException { Map shardStatus = new HashMap<>(); - Repository repository = repositoriesService.repository(snapshotId.getRepository()); - IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(snapshotId.getRepository()); - SnapshotInfo snapshot = repository.readSnapshot(snapshotId); - MetaData metaData = repository.readSnapshotMetaData(snapshotId, snapshot, snapshot.indices()); - for (String index : snapshot.indices()) { + Repository repository = repositoriesService.repository(repositoryName); + IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(repositoryName); + MetaData metaData = repository.readSnapshotMetaData(snapshotInfo, snapshotInfo.indices()); + for (String index : snapshotInfo.indices()) { IndexMetaData indexMetaData = metaData.indices().get(index); if (indexMetaData != null) { int numberOfShards = indexMetaData.getNumberOfShards(); for (int i = 0; i < numberOfShards; i++) { ShardId shardId = new ShardId(indexMetaData.getIndex(), i); - SnapshotShardFailure shardFailure = findShardFailure(snapshot.shardFailures(), shardId); + SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId); if (shardFailure != null) { IndexShardSnapshotStatus shardSnapshotStatus = new IndexShardSnapshotStatus(); shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE); shardSnapshotStatus.failure(shardFailure.reason()); shardStatus.put(shardId, shardSnapshotStatus); } else { - IndexShardSnapshotStatus shardSnapshotStatus = indexShardRepository.snapshotStatus(snapshotId, snapshot.version(), shardId); + IndexShardSnapshotStatus shardSnapshotStatus = + indexShardRepository.snapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId); shardStatus.put(shardId, shardSnapshotStatus); } } @@ -606,15 +646,15 @@ public class SnapshotsService extends AbstractLifecycleComponent failures = new ArrayList<>(); ArrayList shardFailures = new ArrayList<>(); for (ObjectObjectCursor shardStatus : entry.shards()) { @@ -824,11 +864,11 @@ public class SnapshotsService extends AbstractLifecycleComponent listener) { clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() { @Override @@ -863,7 +903,7 @@ public class SnapshotsService extends AbstractLifecycleComponent entries = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshots.entries()) { - if (entry.snapshotId().equals(snapshotId)) { + if (entry.snapshot().equals(snapshot)) { changed = true; } else { entries.add(entry); @@ -879,7 +919,7 @@ public class SnapshotsService extends AbstractLifecycleComponent matchedEntry = repository.snapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst(); + // if nothing found by the same name, then look in the cluster state for current in progress snapshots + if (matchedEntry.isPresent() == false) { + matchedEntry = currentSnapshots(repositoryName, Collections.emptyList()).stream() + .map(e -> e.snapshot().getSnapshotId()).filter(s -> s.getName().equals(snapshotName)).findFirst(); + } + if (matchedEntry.isPresent() == false) { + throw new SnapshotMissingException(repositoryName, snapshotName); + } + deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener); + } + /** * Deletes snapshot from repository. *

* If the snapshot is still running cancels the snapshot first and then deletes it from the repository. * - * @param snapshotId snapshot id - * @param listener listener + * @param snapshot snapshot + * @param listener listener */ - public void deleteSnapshot(final SnapshotId snapshotId, final DeleteSnapshotListener listener) { - validate(snapshotId); + public void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener) { clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask() { boolean waitForSnapshot = false; @@ -926,22 +988,22 @@ public class SnapshotsService extends AbstractLifecycleComponent shards; - if (snapshot.state() == State.STARTED && snapshot.shards() != null) { + if (snapshotEntry.state() == State.STARTED && snapshotEntry.shards() != null) { // snapshot is currently running - stop started shards ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(); - for (ObjectObjectCursor shardEntry : snapshot.shards()) { + for (ObjectObjectCursor shardEntry : snapshotEntry.shards()) { ShardSnapshotStatus status = shardEntry.value; if (!status.state().completed()) { shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED)); @@ -950,14 +1012,14 @@ public class SnapshotsService extends AbstractLifecycleComponent shardStatus : snapshot.shards().values()) { + for (ObjectCursor shardStatus : snapshotEntry.shards().values()) { // Check if we still have shard running on existing nodes if (shardStatus.value.state().completed() == false && shardStatus.value.nodeId() != null && currentState.nodes().get(shardStatus.value.nodeId()) != null) { @@ -972,11 +1034,11 @@ public class SnapshotsService extends AbstractLifecycleComponent { + try { + Repository repository = repositoriesService.repository(snapshot.getRepository()); + repository.deleteSnapshot(snapshot.getSnapshotId()); + listener.onResponse(); + } catch (Throwable t) { + listener.onFailure(t); } }); } @@ -1203,7 +1262,7 @@ public class SnapshotsService extends AbstractLifecycleComponent repoVersions = listRepoVersions(); + // run the test for each supported version + for (final String version : repoVersions) { + final String repoName = "test-repo-" + version; + logger.info("--> creating repository [{}] for version [{}]", repoName, version); + createRepository(version, repoName); + + logger.info("--> get the snapshots"); + final String originalIndex = "index-" + version; + final Set indices = Sets.newHashSet(originalIndex); + final Set snapshotInfos = Sets.newHashSet(getSnapshots(repoName)); + assertThat(snapshotInfos.size(), equalTo(1)); + SnapshotInfo originalSnapshot = snapshotInfos.iterator().next(); + assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", SnapshotId.UNASSIGNED_UUID))); + assertThat(Sets.newHashSet(originalSnapshot.indices()), equalTo(indices)); + + logger.info("--> restore the original snapshot"); + final Set restoredIndices = Sets.newHashSet( + restoreSnapshot(repoName, originalSnapshot.snapshotId().getName()) + ); + assertThat(restoredIndices, equalTo(indices)); + // make sure it has documents + for (final String searchIdx : restoredIndices) { + assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(), greaterThan(0L)); + } + deleteIndices(restoredIndices); // delete so we can restore again later + + final String snapshotName2 = "test_2"; + logger.info("--> take a new snapshot of the old index"); + final int addedDocSize = 10; + for (int i = 0; i < addedDocSize; i++) { + index(originalIndex, "doc", Integer.toString(i), "foo", "new-bar-" + i); + } + refresh(); + snapshotInfos.add(createSnapshot(repoName, snapshotName2)); + + logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName2); + Set snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName)); + assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos)); + snapshotInfosFromRepo.forEach(snapshotInfo -> { + assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices)); + }); + + final String snapshotName3 = "test_3"; + final String indexName2 = "index2"; + logger.info("--> take a new snapshot with a new index"); + createIndex(indexName2); + indices.add(indexName2); + for (int i = 0; i < addedDocSize; i++) { + index(indexName2, "doc", Integer.toString(i), "foo", "new-bar-" + i); + } + refresh(); + snapshotInfos.add(createSnapshot(repoName, snapshotName3)); + + logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName3); + snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName)); + assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos)); + snapshotInfosFromRepo.forEach(snapshotInfo -> { + if (snapshotInfo.snapshotId().getName().equals(snapshotName3)) { + // only the last snapshot has all the indices + assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices)); + } else { + assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(Sets.newHashSet(originalIndex))); + } + }); + deleteIndices(indices); // clean up indices + + logger.info("--> restore the old snapshot again"); + Set oldRestoredIndices = Sets.newHashSet(restoreSnapshot(repoName, originalSnapshot.snapshotId().getName())); + assertThat(oldRestoredIndices, equalTo(Sets.newHashSet(originalIndex))); + for (final String searchIdx : oldRestoredIndices) { + assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(), + greaterThanOrEqualTo((long)addedDocSize)); + } + deleteIndices(oldRestoredIndices); + + logger.info("--> restore the new snapshot"); + Set newSnapshotIndices = Sets.newHashSet(restoreSnapshot(repoName, snapshotName3)); + assertThat(newSnapshotIndices, equalTo(Sets.newHashSet(originalIndex, indexName2))); + for (final String searchIdx : newSnapshotIndices) { + assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(), + greaterThanOrEqualTo((long)addedDocSize)); + } + deleteIndices(newSnapshotIndices); // clean up indices before starting again + } + } + + private List listRepoVersions() throws Exception { + final String prefix = "repo"; + final List repoVersions = new ArrayList<>(); + final Path repoFiles = getBwcIndicesPath(); + try (final DirectoryStream dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) { + for (final Path entry : dirStream) { + final String fileName = entry.getFileName().toString(); + String version = fileName.substring(prefix.length() + 1); + version = version.substring(0, version.length() - ".zip".length()); + repoVersions.add(version); + } + } + return Collections.unmodifiableList(repoVersions); + } + + private void createRepository(final String version, final String repoName) throws Exception { + final String prefix = "repo"; + final Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); + final Path repoPath = randomRepoPath(); + FileTestUtils.unzip(repoFile, repoPath, "repo/"); + assertAcked(client().admin().cluster().preparePutRepository(repoName) + .setType("fs") + .setSettings(Settings.builder().put("location", repoPath))); + } + + private List getSnapshots(final String repoName) throws Exception { + return client().admin().cluster().prepareGetSnapshots(repoName) + .addSnapshots("_all") + .get() + .getSnapshots(); + } + + private SnapshotInfo createSnapshot(final String repoName, final String snapshotName) throws Exception { + return client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get() + .getSnapshotInfo(); + } + + private List restoreSnapshot(final String repoName, final String snapshotName) throws Exception { + return client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get() + .getRestoreInfo() + .indices(); + } + + private void deleteIndices(final Set indices) throws Exception { + client().admin().indices().prepareDelete(indices.toArray(new String[indices.size()])).get(); + } + +} diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index ad16267cde9..419104bfe34 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -40,7 +40,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.lang.reflect.Modifier; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.DirectoryStream; diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 1d013737a30..b82b5e0ba60 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -30,7 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.RepositoriesMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -51,6 +51,7 @@ import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESIntegTestCase; import java.util.Collections; @@ -653,7 +654,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { switch (randomIntBetween(0, 1)) { case 0: return new SnapshotsInProgress(new SnapshotsInProgress.Entry( - new SnapshotId(randomName("repo"), randomName("snap")), + new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), randomBoolean(), randomBoolean(), SnapshotsInProgress.State.fromValue((byte) randomIntBetween(0, 6)), @@ -662,7 +663,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { ImmutableOpenMap.of())); case 1: return new RestoreInProgress(new RestoreInProgress.Entry( - new SnapshotId(randomName("repo"), randomName("snap")), + new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), RestoreInProgress.State.fromValue((byte) randomIntBetween(0, 3)), emptyList(), ImmutableOpenMap.of())); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 31f48a68654..7267252b19f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -20,18 +20,15 @@ package org.elasticsearch.cluster.routing; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; - public class ShardRoutingTests extends ESTestCase { public void testIsSameAllocation() { @@ -155,8 +152,8 @@ public class ShardRoutingTests extends ESTestCase { case 4: // change restore source otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource() == null ? new RestoreSource(new SnapshotId("test", "s1"), Version.CURRENT, "test") : - new RestoreSource(otherRouting.restoreSource().snapshotId(), Version.CURRENT, otherRouting.index() + "_1"), + otherRouting.restoreSource() == null ? new RestoreSource(new Snapshot("test", new SnapshotId("s1", UUIDs.randomBase64UUID())), Version.CURRENT, "test") : + new RestoreSource(otherRouting.restoreSource().snapshot(), Version.CURRENT, otherRouting.index() + "_1"), otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo()); break; case 5: diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index a0ef14ee98d..82703c6f29e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -26,7 +26,8 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; @@ -35,6 +36,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESAllocationTestCase; import java.util.Collections; @@ -130,7 +132,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metaData(metaData) - .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), Version.CURRENT, "test"), new IntHashSet()).build()).build(); + .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"), new IntHashSet()).build()).build(); for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) { assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED)); } @@ -142,7 +144,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metaData(metaData) - .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), Version.CURRENT, "test")).build()).build(); + .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()).build(); for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) { assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED)); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java index 8eb3e2f8a8b..54df74db1f5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java @@ -25,7 +25,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -48,6 +49,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.gateway.NoopGatewayAllocator; @@ -360,7 +362,8 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .metaData(metaData) - .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), + .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), + new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()) .nodes(DiscoveryNodes.builder().put(newNode).put(oldNode1).put(oldNode2)).build(); AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{ diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index ee1cf7280e7..3e088f4f82e 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RestoreSource; @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardStateMetaData; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Before; @@ -339,8 +340,9 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .putActiveAllocationIds(0, hasActiveAllocation ? Sets.newHashSet("allocId") : Collections.emptySet())) .build(); + final Snapshot snapshot = new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())); RoutingTable routingTable = RoutingTable.builder() - .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new SnapshotId("test", "test"), version, shardId.getIndexName())) + .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(snapshot, version, shardId.getIndexName())) .build(); ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT) .metaData(metaData) @@ -419,7 +421,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .build(); RoutingTable routingTable = RoutingTable.builder() - .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new SnapshotId("test", "test"), Version.CURRENT, shardId.getIndexName())) + .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())), Version.CURRENT, shardId.getIndexName())) .build(); ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT) .metaData(metaData) diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 52aa07b3eb3..7774537c734 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -49,7 +49,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.RestoreSource; @@ -97,6 +98,7 @@ import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.FieldMaskingReader; @@ -1110,7 +1112,8 @@ public class IndexShardTests extends ESSingleNodeTestCase { client().admin().indices().prepareFlush("test").get(); // only flush test final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry(); ShardRouting routing = ShardRoutingHelper.reinit(origRouting); - routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test")); + final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID())); + routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(snapshot, Version.CURRENT, "test")); test_target.removeShard(0, "just do it man!"); final IndexShard test_target_shard = test_target.createShard(routing); Store sourceStore = test_shard.store(); diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java new file mode 100644 index 00000000000..3d46c0bbacf --- /dev/null +++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -0,0 +1,230 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.blobstore; + +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.blobId; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.parseNameUUIDFromBlobName; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for the {@link BlobStoreRepository} and its subclasses. + */ +public class BlobStoreRepositoryTests extends ESSingleNodeTestCase { + + public void testRetrieveSnapshots() throws Exception { + final Client client = client(); + final Path location = ESIntegTestCase.randomRepoPath(node().settings()); + final String repositoryName = "test-repo"; + + logger.info("--> creating repository"); + PutRepositoryResponse putRepositoryResponse = + client.admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put(node().settings()).put("location", location)) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + logger.info("--> creating an index and indexing documents"); + final String indexName = "test-idx"; + createIndex(indexName); + ensureGreen(); + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + String id = Integer.toString(i); + client().prepareIndex(indexName, "type1", id).setSource("text", "sometext").get(); + } + client().admin().indices().prepareFlush(indexName).setWaitIfOngoing(true).get(); + + logger.info("--> create first snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, "test-snap-1") + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + logger.info("--> create second snapshot"); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, "test-snap-2") + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + logger.info("--> make sure the node's repository can resolve the snapshots"); + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + @SuppressWarnings("unchecked") final BlobStoreRepository repository = + (BlobStoreRepository) repositoriesService.repository(repositoryName); + final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2); + + List snapshotIds = repository.snapshots().stream() + .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) + .collect(Collectors.toList()); + assertThat(snapshotIds, equalTo(originalSnapshots)); + } + + public void testSnapshotIndexFile() throws Exception { + final Client client = client(); + final Path location = ESIntegTestCase.randomRepoPath(node().settings()); + final String repositoryName = "test-repo"; + + PutRepositoryResponse putRepositoryResponse = + client.admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put(node().settings()).put("location", location)) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + @SuppressWarnings("unchecked") final BlobStoreRepository repository = + (BlobStoreRepository) repositoriesService.repository(repositoryName); + + // write to and read from a snapshot file with no entries + repository.writeSnapshotList(Collections.emptyList()); + List readSnapshotIds = repository.readSnapshotList(); + assertThat(readSnapshotIds.size(), equalTo(0)); + + // write to and read from a snapshot file with a random number of entries + final int numSnapshots = randomIntBetween(1, 1000); + final List snapshotIds = new ArrayList<>(numSnapshots); + for (int i = 0; i < numSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotList(snapshotIds); + readSnapshotIds = repository.readSnapshotList(); + assertThat(readSnapshotIds, equalTo(snapshotIds)); + } + + public void testOldIndexFileFormat() throws Exception { + final Client client = client(); + final Path location = ESIntegTestCase.randomRepoPath(node().settings()); + final String repositoryName = "test-repo"; + + PutRepositoryResponse putRepositoryResponse = + client.admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put(node().settings()).put("location", location)) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + @SuppressWarnings("unchecked") final BlobStoreRepository repository = + (BlobStoreRepository) repositoriesService.repository(repositoryName); + + // write old index file format + final int numOldSnapshots = randomIntBetween(1, 50); + final List snapshotIds = new ArrayList<>(); + for (int i = 0; i < numOldSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID)); + } + writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList())); + List readSnapshotIds = repository.readSnapshotList(); + assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds))); + + // write to and read from a snapshot file with a random number of new entries added + final int numSnapshots = randomIntBetween(1, 1000); + for (int i = 0; i < numSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotList(snapshotIds); + readSnapshotIds = repository.readSnapshotList(); + assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds))); + } + + public void testParseUUIDFromBlobName() { + String blobStr = "abc123"; + Tuple pair = parseNameUUIDFromBlobName(blobStr); + assertThat(pair.v1(), equalTo(blobStr)); // snapshot name + assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); // snapshot uuid + blobStr = "abcefghijklmnopqrstuvwxyz"; + pair = parseNameUUIDFromBlobName(blobStr); + assertThat(pair.v1(), equalTo(blobStr)); + assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); + blobStr = "abc123-xyz"; // not enough characters after '-' to have a uuid + pair = parseNameUUIDFromBlobName(blobStr); + assertThat(pair.v1(), equalTo(blobStr)); + assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); + blobStr = "abc123-a1b2c3d4e5f6g7h8i9j0k1"; + pair = parseNameUUIDFromBlobName(blobStr); + assertThat(pair.v1(), equalTo("abc123")); + assertThat(pair.v2(), equalTo("a1b2c3d4e5f6g7h8i9j0k1")); + } + + public void testBlobId() { + SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID); + assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name + snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID); + assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name + String uuid = UUIDs.randomBase64UUID(); + snapshotId = new SnapshotId("abc123", uuid); + assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid + uuid = UUIDs.randomBase64UUID(); + snapshotId = new SnapshotId("abc-123", uuid); + assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid + } + + private void writeOldFormat(final BlobStoreRepository repository, final List snapshotNames) throws Exception { + final BytesReference bRef; + try (BytesStreamOutput bStream = new BytesStreamOutput()) { + try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); + builder.startObject(); + builder.startArray("snapshots"); + for (final String snapshotName : snapshotNames) { + builder.value(snapshotName); + } + builder.endArray(); + builder.endObject(); + builder.close(); + } + bRef = bStream.bytes(); + } + repository.blobContainer().writeBlob(BlobStoreRepository.SNAPSHOTS_FILE, bRef); // write to index file + } + +} diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java index 978d226da40..b603ded8697 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java @@ -21,7 +21,8 @@ package org.elasticsearch.rest.action.cat; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.common.Randomness; @@ -32,6 +33,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.rest.RestController; +import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -76,10 +78,9 @@ public class RestRecoveryActionTests extends ESTestCase { final RestoreSource restoreSource = randomBoolean() ? mock(RestoreSource.class) : null; if (restoreSource != null) { - final SnapshotId snapshotId = mock(SnapshotId.class); - when(snapshotId.getRepository()).thenReturn(randomAsciiOfLength(8)); - when(snapshotId.getSnapshot()).thenReturn(randomAsciiOfLength(8)); - when(restoreSource.snapshotId()).thenReturn(snapshotId); + final Snapshot snapshot = new Snapshot(randomAsciiOfLength(8), + new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + when(restoreSource.snapshot()).thenReturn(snapshot); } RecoveryState.Index index = mock(RecoveryState.Index.class); @@ -166,10 +167,10 @@ public class RestRecoveryActionTests extends ESTestCase { assertThat(cells.get(8).value, equalTo(state.getTargetNode().getName())); assertThat( cells.get(9).value, - equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository())); + equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository())); assertThat( cells.get(10).value, - equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot())); + equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName())); assertThat(cells.get(11).value, equalTo(state.getIndex().totalRecoverFiles())); assertThat(cells.get(12).value, equalTo(state.getIndex().recoveredFileCount())); assertThat(cells.get(13).value, equalTo(percent(state.getIndex().recoveredFilesPercent()))); diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 102173626f3..50fb3f9074b 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.PendingClusterTask; @@ -108,18 +107,29 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { fail("Timeout!!!"); } - public SnapshotInfo waitForCompletion(String repository, String snapshot, TimeValue timeout) throws InterruptedException { + public SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException { long start = System.currentTimeMillis(); - SnapshotId snapshotId = new SnapshotId(repository, snapshot); while (System.currentTimeMillis() - start < timeout.millis()) { - List snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshot).get().getSnapshots(); + List snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName).get().getSnapshots(); assertThat(snapshotInfos.size(), equalTo(1)); if (snapshotInfos.get(0).state().completed()) { // Make sure that snapshot clean up operations are finished ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get(); SnapshotsInProgress snapshotsInProgress = stateResponse.getState().custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress == null || snapshotsInProgress.snapshot(snapshotId) == null) { + if (snapshotsInProgress == null) { return snapshotInfos.get(0); + } else { + boolean found = false; + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { + final Snapshot curr = entry.snapshot(); + if (curr.getRepository().equals(repository) && curr.getSnapshotId().getName().equals(snapshotName)) { + found = true; + break; + } + } + if (found == false) { + return snapshotInfos.get(0); + } } } Thread.sleep(100); @@ -128,12 +138,13 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { return null; } - public static String blockNodeWithIndex(String index) { - for(String node : internalCluster().nodesInclude("test-idx")) { - ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository("test-repo")).blockOnDataFiles(true); + public static String blockNodeWithIndex(final String repositoryName, final String indexName) { + for(String node : internalCluster().nodesInclude(indexName)) { + ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repositoryName)) + .blockOnDataFiles(true); return node; } - fail("No nodes for the index " + index + " found"); + fail("No nodes for the index " + indexName + " found"); return null; } @@ -163,8 +174,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { } } - public static void unblockNode(String node) { - ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository("test-repo")).unblock(); + public static void unblockNode(final String repository, final String node) { + ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repository)).unblock(); } protected void assertBusyPendingTasks(final String taskPrefix, final int expectedCount) throws Exception { diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 34333a583af..3ba6c875b68 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -313,7 +313,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); // Pick one node and block it - String blockedNode = blockNodeWithIndex("test-idx"); + String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); @@ -322,7 +322,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60)); logger.info("--> execution was blocked on node [{}], shutting it down", blockedNode); - unblockNode(blockedNode); + unblockNode("test-repo", blockedNode); logger.info("--> stopping node [{}]", blockedNode); stopNode(blockedNode); @@ -361,7 +361,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); // Pick one node and block it - String blockedNode = blockNodeWithIndex("test-idx"); + String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); // Remove it from the list of available nodes nodes.remove(blockedNode); @@ -377,7 +377,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest ListenableActionFuture deleteSnapshotResponseFuture = internalCluster().client(nodes.get(0)).admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").execute(); // Make sure that abort makes some progress Thread.sleep(100); - unblockNode(blockedNode); + unblockNode("test-repo", blockedNode); logger.info("--> stopping node [{}]", blockedNode); stopNode(blockedNode); try { diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 3996aad25e3..19b46710fea 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -50,7 +50,6 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; -import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -73,9 +72,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -917,7 +918,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); logger.info("--> truncate snapshot file to make it unreadable"); - Path snapshotPath = repo.resolve("snap-test-snap-1.dat"); + Path snapshotPath = repo.resolve("snap-test-snap-1-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat"); try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } @@ -1120,7 +1121,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it - String blockedNode = blockNodeWithIndex("test-idx"); + String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); @@ -1133,7 +1134,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas client().admin().indices().prepareUpdateSettings("test-idx").setSettings(excludeSettings).get(); logger.info("--> unblocking blocked node"); - unblockNode(blockedNode); + unblockNode("test-repo", blockedNode); logger.info("--> waiting for completion"); SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600)); logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size()); @@ -1183,7 +1184,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it - String blockedNode = blockNodeWithIndex("test-idx"); + String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); @@ -1215,7 +1216,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setType("fs").setSettings(Settings.builder().put("location", repositoryLocation.resolve("test")))); logger.info("--> unblocking blocked node"); - unblockNode(blockedNode); + unblockNode("test-repo", blockedNode); logger.info("--> waiting for completion"); SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600)); logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size()); @@ -1436,7 +1437,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); // Pick one node and block it - String blockedNode = blockNodeWithIndex("test-idx"); + String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); @@ -1477,7 +1478,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS)); logger.info("--> unblocking blocked node"); - unblockNode(blockedNode); + unblockNode("test-repo", blockedNode); snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600)); logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size()); @@ -1904,7 +1905,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } finally { if (initBlocking) { logger.info("--> unblock running master node"); - unblockNode(internalCluster().getMasterName()); + unblockNode("test-repo", internalCluster().getMasterName()); } else { logger.info("--> unblock all data nodes"); unblockAllDataNodes("test-repo"); @@ -1993,14 +1994,17 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Client client = client(); logger.info("--> creating repository"); - assertAcked(client.admin().cluster().preparePutRepository("test-repo") + final String repositoryName = "test-repo"; + assertAcked(client.admin().cluster().preparePutRepository(repositoryName) .setType("mock").setSettings(Settings.builder() .put("location", randomRepoPath()) .put("compress", randomBoolean()) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) )); - createIndex("test-idx"); + logger.info("--> create the index"); + final String idxName = "test-idx"; + createIndex(idxName); ensureGreen(); ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName()); @@ -2008,7 +2012,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final CountDownLatch countDownLatch = new CountDownLatch(1); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); + final String snapshotName = "test-snap"; + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); @@ -2020,11 +2025,18 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public ClusterState execute(ClusterState currentState) { // Simulate orphan snapshot ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder(); - shards.put(new ShardId("test-idx", "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED)); - shards.put(new ShardId("test-idx", "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED)); - shards.put(new ShardId("test-idx", "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED)); + shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED)); + shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED)); + shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED)); List entries = new ArrayList<>(); - entries.add(new Entry(new SnapshotId("test-repo", "test-snap"), true, false, State.ABORTED, Collections.singletonList("test-idx"), System.currentTimeMillis(), shards.build())); + entries.add(new Entry(new Snapshot(repositoryName, + createSnapshotResponse.getSnapshotInfo().snapshotId()), + true, + false, + State.ABORTED, + Collections.singletonList(idxName), + System.currentTimeMillis(), + shards.build())); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build(); } @@ -2042,8 +2054,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas countDownLatch.await(); logger.info("--> try deleting the orphan snapshot"); - assertAcked(client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get("10s")); - + assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s")); } private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException { @@ -2143,33 +2154,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .put("compress", randomBoolean()) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); - try { - client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get(); - fail("shouldn't be here"); - } catch (InvalidSnapshotNameException ex) { - assertThat(ex.getMessage(), containsString("Invalid snapshot name")); - } - - try { - client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get(); - fail("shouldn't be here"); - } catch (InvalidSnapshotNameException ex) { - assertThat(ex.getMessage(), containsString("Invalid snapshot name")); - } - - try { - client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get(); - fail("shouldn't be here"); - } catch (InvalidSnapshotNameException ex) { - assertThat(ex.getMessage(), containsString("Invalid snapshot name")); - } - - try { - client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("_foo").get(); - fail("shouldn't be here"); - } catch (InvalidSnapshotNameException ex) { - assertThat(ex.getMessage(), containsString("Invalid snapshot name")); - } + expectThrows(InvalidSnapshotNameException.class, + () -> client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get()); + expectThrows(SnapshotMissingException.class, + () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get()); + expectThrows(SnapshotMissingException.class, + () -> client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get()); + expectThrows(SnapshotMissingException.class, + () -> client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("_foo").get()); } public void testListCorruptedSnapshot() throws Exception { @@ -2199,7 +2191,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); logger.info("--> truncate snapshot file to make it unreadable"); - Path snapshotPath = repo.resolve("snap-test-snap-2.dat"); + Path snapshotPath = repo.resolve("snap-test-snap-2-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat"); try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } @@ -2211,13 +2203,194 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(snapshotInfos.size(), equalTo(1)); assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfos.get(0).name(), equalTo("test-snap-1")); + assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap-1")); try { client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get().getSnapshots(); } catch (SnapshotException ex) { - assertThat(ex.snapshot().getRepository(), equalTo("test-repo")); - assertThat(ex.snapshot().getSnapshot(), equalTo("test-snap-2")); + assertThat(ex.getRepositoryName(), equalTo("test-repo")); + assertThat(ex.getSnapshotName(), equalTo("test-snap-2")); } } + + public void testCannotCreateSnapshotsWithSameName() throws Exception { + final String repositoryName = "test-repo"; + final String snapshotName = "test-snap"; + final String indexName = "test-idx"; + final Client client = client(); + final Path repo = randomRepoPath(); + + logger.info("--> creating repository at {}", repo.toAbsolutePath()); + assertAcked(client.admin().cluster().preparePutRepository(repositoryName) + .setType("fs").setSettings(Settings.builder() + .put("location", repo) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + logger.info("--> creating an index and indexing documents"); + createIndex(indexName); + ensureGreen(); + for (int i = 0; i < 10; i++) { + index(indexName, "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + logger.info("--> take first snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, snapshotName) + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + logger.info("--> index more documents"); + for (int i = 10; i < 20; i++) { + index(indexName, "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + logger.info("--> second snapshot of the same name should fail"); + try { + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, snapshotName) + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + fail("should not be allowed to create a snapshot with the same name as an already existing snapshot: " + + createSnapshotResponse.getSnapshotInfo().snapshotId()); + } catch (SnapshotCreationException e) { + assertThat(e.getMessage(), containsString("snapshot with the same name already exists")); + } + + logger.info("--> delete the first snapshot"); + client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get(); + + logger.info("--> try creating a snapshot with the same name, now it should work because the first one was deleted"); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, snapshotName) + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().snapshotId().getName(), equalTo(snapshotName)); + } + + public void testGetSnapshotsRequest() throws Exception { + final String repositoryName = "test-repo"; + final String indexName = "test-idx"; + final Client client = client(); + final Path repo = randomRepoPath(); + + logger.info("--> creating repository at {}", repo.toAbsolutePath()); + assertAcked(client.admin().cluster().preparePutRepository(repositoryName) + .setType("mock").setSettings(Settings.builder() + .put("location", repo) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put("wait_after_unblock", 200))); + + logger.info("--> get snapshots on an empty repository"); + expectThrows(SnapshotMissingException.class, () -> client.admin() + .cluster() + .prepareGetSnapshots(repositoryName) + .addSnapshots("non-existent-snapshot") + .get()); + // with ignore unavailable set to true, should not throw an exception + GetSnapshotsResponse getSnapshotsResponse = client.admin() + .cluster() + .prepareGetSnapshots(repositoryName) + .setIgnoreUnavailable(true) + .addSnapshots("non-existent-snapshot") + .get(); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); + + logger.info("--> creating an index and indexing documents"); + // Create index on 2 nodes and make sure each node has a primary by setting no replicas + assertAcked(prepareCreate(indexName, 1, Settings.builder().put("number_of_replicas", 0))); + ensureGreen(); + for (int i = 0; i < 10; i++) { + index(indexName, "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final int numSnapshots = randomIntBetween(1, 3) + 1; + logger.info("--> take {} snapshot(s)", numSnapshots); + final String[] snapshotNames = new String[numSnapshots]; + for (int i = 0; i < numSnapshots - 1; i++) { + final String snapshotName = randomAsciiOfLength(8).toLowerCase(Locale.ROOT); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repositoryName, snapshotName) + .setWaitForCompletion(true) + .setIndices(indexName) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + snapshotNames[i] = snapshotName; + } + logger.info("--> take another snapshot to be in-progress"); + // add documents so there are data files to block on + for (int i = 10; i < 20; i++) { + index(indexName, "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String inProgressSnapshot = randomAsciiOfLength(8).toLowerCase(Locale.ROOT); + snapshotNames[numSnapshots - 1] = inProgressSnapshot; + // block a node so the create snapshot operation can remain in progress + final String blockedNode = blockNodeWithIndex(repositoryName, indexName); + client.admin().cluster().prepareCreateSnapshot(repositoryName, inProgressSnapshot) + .setWaitForCompletion(false) + .setIndices(indexName) + .get(); + waitForBlock(blockedNode, repositoryName, TimeValue.timeValueSeconds(60)); // wait for block to kick in + + logger.info("--> get all snapshots with a current in-progress"); + // with ignore unavailable set to true, should not throw an exception + getSnapshotsResponse = client.admin().cluster() + .prepareGetSnapshots(repositoryName) + .addSnapshots("_all") + .get(); + List sortedNames = Arrays.asList(snapshotNames); + Collections.sort(sortedNames); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots().stream() + .map(s -> s.snapshotId().getName()) + .sorted() + .collect(Collectors.toList()), equalTo(sortedNames)); + + getSnapshotsResponse = client.admin().cluster() + .prepareGetSnapshots(repositoryName) + .addSnapshots(snapshotNames) + .get(); + sortedNames = Arrays.asList(snapshotNames); + Collections.sort(sortedNames); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots().stream() + .map(s -> s.snapshotId().getName()) + .sorted() + .collect(Collectors.toList()), equalTo(sortedNames)); + + logger.info("--> make sure duplicates are not returned in the response"); + String regexName = snapshotNames[randomIntBetween(0, numSnapshots - 1)]; + final int splitPos = regexName.length() / 2; + final String firstRegex = regexName.substring(0, splitPos) + "*"; + final String secondRegex = "*" + regexName.substring(splitPos); + getSnapshotsResponse = client.admin().cluster() + .prepareGetSnapshots(repositoryName) + .addSnapshots(snapshotNames) + .addSnapshots(firstRegex, secondRegex) + .get(); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots)); + assertThat(getSnapshotsResponse.getSnapshots().stream() + .map(s -> s.snapshotId().getName()) + .sorted() + .collect(Collectors.toList()), equalTo(sortedNames)); + + unblockNode(repositoryName, blockedNode); // unblock node + waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60)); + } + } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java index e6379468382..31c0a193f07 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java @@ -86,7 +86,7 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase { counts[i] = client().prepareSearch(indices[i]).setSize(0).get().getHits().totalHits(); } - logger.info("--> snapshot subset of indices before upgrage"); + logger.info("--> snapshot subset of indices before upgrade"); CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("index_before_*").get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java new file mode 100644 index 00000000000..cb297785e4b --- /dev/null +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.hamcrest.CoreMatchers.equalTo; + +/** + * Tests for the {@link Snapshot} class. + */ +public class SnapshotTests extends ESTestCase { + + public void testSnapshotEquals() { + final SnapshotId snapshotId = new SnapshotId("snap", UUIDs.randomBase64UUID()); + final Snapshot original = new Snapshot("repo", snapshotId); + final Snapshot expected = new Snapshot(original.getRepository(), original.getSnapshotId()); + assertThat(expected, equalTo(original)); + assertThat(expected.getRepository(), equalTo(original.getRepository())); + assertThat(expected.getSnapshotId(), equalTo(original.getSnapshotId())); + assertThat(expected.getSnapshotId().getName(), equalTo(original.getSnapshotId().getName())); + assertThat(expected.getSnapshotId().getUUID(), equalTo(original.getSnapshotId().getUUID())); + } + + public void testSerialization() throws IOException { + final SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(randomIntBetween(2, 8)), UUIDs.randomBase64UUID()); + final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId); + final BytesStreamOutput out = new BytesStreamOutput(); + original.writeTo(out); + final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); + assertThat(new Snapshot(in), equalTo(original)); + } + +} diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index 7df44738076..8b02f90f51c 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -21,7 +21,7 @@ package org.elasticsearch.snapshots.mockstore; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetaData; @@ -174,6 +174,10 @@ public class MockRepository extends FsRepository { blockOnControlFiles = blocked; } + public boolean blockOnDataFiles() { + return blockOnDataFiles; + } + public synchronized void unblockExecution() { blocked = false; // Clean blocking flags, so we wouldn't try to block again diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 8af614df605..4d3459cdcd4 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -24,7 +24,7 @@ import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -166,7 +166,7 @@ public class AzureRepository extends BlobStoreRepository { super.initializeSnapshot(snapshotId, indices, metaData); } catch (StorageException | URISyntaxException e) { logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage()); - throw new SnapshotCreationException(snapshotId, e); + throw new SnapshotCreationException(repositoryName, snapshotId, e); } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java index 5a1c76df413..8160c560325 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java @@ -94,8 +94,15 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) { MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); for (String blobName : blobs.keySet()) { - if (startsWithIgnoreCase(blobName, prefix)) { - blobsBuilder.put(blobName, new PlainBlobMetaData(blobName, blobs.get(blobName).size())); + final String checkBlob; + if (keyPath != null) { + // strip off key path from the beginning of the blob name + checkBlob = blobName.replace(keyPath, ""); + } else { + checkBlob = blobName; + } + if (startsWithIgnoreCase(checkBlob, prefix)) { + blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size())); } } return blobsBuilder.immutableMap(); diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java index 50d677c600c..98155967514 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java @@ -19,11 +19,15 @@ package org.elasticsearch.common.io; +import org.elasticsearch.common.Nullable; import org.junit.Assert; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.hamcrest.CoreMatchers.equalTo; @@ -50,4 +54,44 @@ public class FileTestUtils { Assert.assertThat(fileContent.trim(), equalTo(expected.trim())); } } + + /** + * Unzip a zip file to a destination directory. If the zip file does not exist, an IOException is thrown. + * If the destination directory does not exist, it will be created. + * + * @param zip zip file to unzip + * @param destDir directory to unzip the file to + * @param prefixToRemove the (optional) prefix in the zip file path to remove when writing to the destination directory + * @throws IOException if zip file does not exist, or there was an error reading from the zip file or + * writing to the destination directory + */ + public static void unzip(final Path zip, final Path destDir, @Nullable final String prefixToRemove) throws IOException { + if (Files.notExists(zip)) { + throw new IOException("[" + zip + "] zip file must exist"); + } + Files.createDirectories(destDir); + + try (final ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) { + ZipEntry entry; + while ((entry = zipInput.getNextEntry()) != null) { + final String entryPath; + if (prefixToRemove != null) { + if (entry.getName().startsWith(prefixToRemove)) { + entryPath = entry.getName().substring(prefixToRemove.length()); + } else { + throw new IOException("prefix not found: " + prefixToRemove); + } + } else { + entryPath = entry.getName(); + } + final Path path = Paths.get(destDir.toString(), entryPath); + if (entry.isDirectory()) { + Files.createDirectories(path); + } else { + Files.copy(zipInput, path); + } + zipInput.closeEntry(); + } + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index c86c111f645..f4d5b4dd482 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -54,6 +54,7 @@ import org.junit.Before; import org.junit.BeforeClass; import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -171,14 +172,15 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { } private Node newNode() { + final Path tempDir = createTempDir(); Settings settings = Settings.builder() .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) // TODO: use a consistent data path for custom paths // This needs to tie into the ESIntegTestCase#indexSettings() method .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent()) .put("node.name", nodeName()) - .put("script.inline", "true") .put("script.stored", "true") .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created