honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhances get snapshots API to allow retrieving repository index only (#24477) 

Currently, the get snapshots API (e.g. /_snapshot/{repositoryName}/_all)
provides information about snapshots in the repository, including the
snapshot state, number of shards snapshotted, failures, etc.  In order
to provide information about each snapshot in the repository, the call
must read the snapshot metadata blob (`snap-{snapshot_uuid}.dat`) for
every snapshot.  In cloud-based repositories, this can be expensive,
both from a cost and performance perspective.  Sometimes, all the user
wants is to retrieve all the names/uuids of each snapshot, and the
indices that went into each snapshot, without any of the other status
information about the snapshot.  This minimal information can be
retrieved from the repository index blob (`index-N`) without needing to
read each snapshot metadata blob.

This commit enhances the get snapshots API with an optional `verbose`
parameter.  If `verbose` is set to false on the request, then the get
snapshots API will only retrieve the minimal information about each
snapshot (the name, uuid, and indices in the snapshot), and only read
this information from the repository index blob, thereby giving users
the option to retrieve the snapshots in a repository in a more
cost-effective and efficient manner.

Closes #24288
This commit is contained in:
Ali Beyad 2017-05-10 15:48:40 -04:00 committed by GitHub
parent fbf532a626
commit 743217a430
17 changed files with 499 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import java.io.IOException;
import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.snapshots.SnapshotInfo.VERBOSE_INTRODUCED;
/**
* Get snapshot request
@ -43,6 +44,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
private boolean ignoreUnavailable;
private boolean verbose = true;
public GetSnapshotsRequest() {
}
@ -123,6 +126,7 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
this.ignoreUnavailable = ignoreUnavailable;
return this;
}
/**
* @return Whether snapshots should be ignored when unavailable (corrupt or temporarily not fetchable)
*/
@ -130,12 +134,36 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
return ignoreUnavailable;
}
/**
* Set to {@code false} to only show the snapshot names and the indices they contain.
* This is useful when the snapshots belong to a cloud-based repository where each
* blob read is a concern (cost wise and performance wise), as the snapshot names and
* indices they contain can be retrieved from a single index blob in the repository,
* whereas the rest of the information requires reading a snapshot metadata file for
* each snapshot requested. Defaults to {@code true}, which returns all information
* about each requested snapshot.
*/
public GetSnapshotsRequest verbose(boolean verbose) {
this.verbose = verbose;
return this;
}
/**
* Returns whether the request will return a verbose response.
*/
public boolean verbose() {
return verbose;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
repository = in.readString();
snapshots = in.readStringArray();
ignoreUnavailable = in.readBoolean();
if (in.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
verbose = in.readBoolean();
}
}
@Override
@ -144,5 +172,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
out.writeString(repository);
out.writeStringArray(snapshots);
out.writeBoolean(ignoreUnavailable);
if (out.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
out.writeBoolean(verbose);
}
}
}

14
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
View File

@ -96,4 +96,18 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde
return this;
}
/**
* Set to {@code false} to only show the snapshot names and the indices they contain.
* This is useful when the snapshots belong to a cloud-based repository where each
* blob read is a concern (cost wise and performance wise), as the snapshot names and
* indices they contain can be retrieved from a single index blob in the repository,
* whereas the rest of the information requires reading a snapshot metadata file for
* each snapshot requested. Defaults to {@code true}, which returns all information
* about each requested snapshot.
*/
public GetSnapshotsRequestBuilder setVerbose(boolean verbose) {
request.verbose(verbose);
return this;
}
}

68
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.action.admin.cluster.snapshots.get;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@ -30,6 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
@ -39,11 +41,13 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Transport Action for get snapshots operation
@ -76,31 +80,35 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
}
@Override
protected void masterOperation(final GetSnapshotsRequest request, ClusterState state,
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state,
final ActionListener<GetSnapshotsResponse> listener) {
try {
final String repository = request.repository();
List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
final List<SnapshotId> currentSnapshotIds = new ArrayList<>();
final RepositoryData repositoryData = snapshotsService.getRepositoryData(repository);
final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
SnapshotId snapshotId = snapshotInfo.snapshotId();
allSnapshotIds.put(snapshotId.getName(), snapshotId);
currentSnapshotIds.add(snapshotId);
currentSnapshots.add(snapshotInfo);
}
final RepositoryData repositoryData;
if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
repositoryData = snapshotsService.getRepositoryData(repository);
for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
allSnapshotIds.put(snapshotId.getName(), snapshotId);
}
} else {
repositoryData = null;
}
final Set<SnapshotId> toResolve = new HashSet<>();
if (isAllSnapshots(request.snapshots())) {
toResolve.addAll(allSnapshotIds.values());
} else {
for (String snapshotOrPattern : request.snapshots()) {
if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
toResolve.addAll(currentSnapshotIds);
toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
} else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
if (allSnapshotIds.containsKey(snapshotOrPattern)) {
toResolve.add(allSnapshotIds.get(snapshotOrPattern));
@ -121,9 +129,23 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
}
}
snapshotInfoBuilder.addAll(snapshotsService.snapshots(
repository, new ArrayList<>(toResolve), repositoryData.getIncompatibleSnapshotIds(), request.ignoreUnavailable()));
listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
final List<SnapshotInfo> snapshotInfos;
if (request.verbose()) {
final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ?
new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet();
snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve),
incompatibleSnapshots, request.ignoreUnavailable());
} else {
if (repositoryData != null) {
// want non-current snapshots as well, which are found in the repository data
snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
} else {
// only want current snapshots
snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
CollectionUtil.timSort(snapshotInfos);
}
}
listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
} catch (Exception e) {
listener.onFailure(e);
}
@ -136,4 +158,32 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
private boolean isCurrentSnapshotsOnly(String[] snapshots) {
return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0]));
}
private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve,
final RepositoryData repositoryData,
final List<SnapshotInfo> currentSnapshots) {
List<SnapshotInfo> snapshotInfos = new ArrayList<>();
for (SnapshotInfo snapshotInfo : currentSnapshots) {
if (toResolve.remove(snapshotInfo.snapshotId())) {
snapshotInfos.add(snapshotInfo.basic());
}
}
Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>();
for (IndexId indexId : repositoryData.getIndices().values()) {
for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
if (toResolve.contains(snapshotId)) {
snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>())
.add(indexId.getName());
}
}
}
for (Map.Entry<SnapshotId, List<String>> entry : snapshotsToIndices.entrySet()) {
final List<String> indices = entry.getValue();
CollectionUtil.timSort(indices);
final SnapshotId snapshotId = entry.getKey();
snapshotInfos.add(new SnapshotInfo(snapshotId, indices, repositoryData.getSnapshotState(snapshotId)));
}
CollectionUtil.timSort(snapshotInfos);
return Collections.unmodifiableList(snapshotInfos);
}
}

171
core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View File

@ -20,14 +20,17 @@
package org.elasticsearch.repositories;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotState;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
@ -51,8 +54,8 @@ public final class RepositoryData {
/**
* An instance initialized for an empty repository.
*/
public static final RepositoryData EMPTY =
new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), Collections.emptyMap(), Collections.emptyList());
public static final RepositoryData EMPTY = new RepositoryData(EMPTY_REPO_GEN,
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList());
/**
* The generational id of the index file from which the repository data was read.
@ -61,7 +64,11 @@ public final class RepositoryData {
/**
* The ids of the snapshots in the repository.
*/
private final List<SnapshotId> snapshotIds;
private final Map<String, SnapshotId> snapshotIds;
/**
* The states of each snapshot in the repository.
*/
private final Map<String, SnapshotState> snapshotStates;
/**
* The indices found in the repository across all snapshots, as a name to {@link IndexId} mapping
*/
@ -75,19 +82,22 @@ public final class RepositoryData {
*/
private final List<SnapshotId> incompatibleSnapshotIds;
public RepositoryData(long genId, List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots,
public RepositoryData(long genId,
Map<String, SnapshotId> snapshotIds,
Map<String, SnapshotState> snapshotStates,
Map<IndexId, Set<SnapshotId>> indexSnapshots,
List<SnapshotId> incompatibleSnapshotIds) {
this.genId = genId;
this.snapshotIds = Collections.unmodifiableList(snapshotIds);
this.indices = Collections.unmodifiableMap(indexSnapshots.keySet()
.stream()
.collect(Collectors.toMap(IndexId::getName, Function.identity())));
this.snapshotIds = Collections.unmodifiableMap(snapshotIds);
this.snapshotStates = Collections.unmodifiableMap(snapshotStates);
this.indices = Collections.unmodifiableMap(indexSnapshots.keySet().stream()
.collect(Collectors.toMap(IndexId::getName, Function.identity())));
this.indexSnapshots = Collections.unmodifiableMap(indexSnapshots);
this.incompatibleSnapshotIds = Collections.unmodifiableList(incompatibleSnapshotIds);
}
protected RepositoryData copy() {
return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
return new RepositoryData(genId, snapshotIds, snapshotStates, indexSnapshots, incompatibleSnapshotIds);
}
/**
@ -98,17 +108,17 @@ public final class RepositoryData {
}
/**
* Returns an unmodifiable list of the snapshot ids.
* Returns an unmodifiable collection of the snapshot ids.
*/
public List<SnapshotId> getSnapshotIds() {
return snapshotIds;
public Collection<SnapshotId> getSnapshotIds() {
return Collections.unmodifiableCollection(snapshotIds.values());
}
/**
* Returns an immutable collection of the snapshot ids in the repository that are incompatible with the
* current ES version.
*/
public List<SnapshotId> getIncompatibleSnapshotIds() {
public Collection<SnapshotId> getIncompatibleSnapshotIds() {
return incompatibleSnapshotIds;
}
@ -116,13 +126,22 @@ public final class RepositoryData {
* Returns an immutable collection of all the snapshot ids in the repository, both active and
* incompatible snapshots.
*/
public List<SnapshotId> getAllSnapshotIds() {
public Collection<SnapshotId> getAllSnapshotIds() {
List<SnapshotId> allSnapshotIds = new ArrayList<>(snapshotIds.size() + incompatibleSnapshotIds.size());
allSnapshotIds.addAll(snapshotIds);
allSnapshotIds.addAll(snapshotIds.values());
allSnapshotIds.addAll(incompatibleSnapshotIds);
return Collections.unmodifiableList(allSnapshotIds);
}
/**
* Returns the {@link SnapshotState} for the given snapshot. Returns {@code null} if
* there is no state for the snapshot.
*/
@Nullable
public SnapshotState getSnapshotState(final SnapshotId snapshotId) {
return snapshotStates.get(snapshotId.getUUID());
}
/**
* Returns an unmodifiable map of the index names to {@link IndexId} in the repository.
*/
@ -134,15 +153,19 @@ public final class RepositoryData {
* Add a snapshot and its indices to the repository; returns a new instance. If the snapshot
* already exists in the repository data, this method throws an IllegalArgumentException.
*/
public RepositoryData addSnapshot(final SnapshotId snapshotId, final List<IndexId> snapshottedIndices) {
if (snapshotIds.contains(snapshotId)) {
public RepositoryData addSnapshot(final SnapshotId snapshotId,
final SnapshotState snapshotState,
final List<IndexId> snapshottedIndices) {
if (snapshotIds.containsKey(snapshotId.getUUID())) {
// if the snapshot id already exists in the repository data, it means an old master
// that is blocked from the cluster is trying to finalize a snapshot concurrently with
// the new master, so we make the operation idempotent
return this;
}
List<SnapshotId> snapshots = new ArrayList<>(snapshotIds);
snapshots.add(snapshotId);
Map<String, SnapshotId> snapshots = new HashMap<>(snapshotIds);
snapshots.put(snapshotId.getUUID(), snapshotId);
Map<String, SnapshotState> newSnapshotStates = new HashMap<>(snapshotStates);
newSnapshotStates.put(snapshotId.getUUID(), snapshotState);
Map<IndexId, Set<SnapshotId>> allIndexSnapshots = new HashMap<>(indexSnapshots);
for (final IndexId indexId : snapshottedIndices) {
if (allIndexSnapshots.containsKey(indexId)) {
@ -158,17 +181,18 @@ public final class RepositoryData {
allIndexSnapshots.put(indexId, ids);
}
}
return new RepositoryData(genId, snapshots, allIndexSnapshots, incompatibleSnapshotIds);
return new RepositoryData(genId, snapshots, newSnapshotStates, allIndexSnapshots, incompatibleSnapshotIds);
}
/**
* Remove a snapshot and remove any indices that no longer exist in the repository due to the deletion of the snapshot.
*/
public RepositoryData removeSnapshot(final SnapshotId snapshotId) {
List<SnapshotId> newSnapshotIds = snapshotIds
.stream()
.filter(id -> snapshotId.equals(id) == false)
.collect(Collectors.toList());
Map<String, SnapshotId> newSnapshotIds = snapshotIds.values().stream()
.filter(id -> snapshotId.equals(id) == false)
.collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
Map<String, SnapshotState> newSnapshotStates = new HashMap<>(snapshotStates);
newSnapshotStates.remove(snapshotId.getUUID());
Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
for (final IndexId indexId : indices.values()) {
Set<SnapshotId> set;
@ -176,7 +200,8 @@ public final class RepositoryData {
assert snapshotIds != null;
if (snapshotIds.contains(snapshotId)) {
if (snapshotIds.size() == 1) {
// removing the snapshot will mean no more snapshots have this index, so just skip over it
// removing the snapshot will mean no more snapshots
// have this index, so just skip over it
continue;
}
set = new LinkedHashSet<>(snapshotIds);
@ -187,21 +212,7 @@ public final class RepositoryData {
indexSnapshots.put(indexId, set);
}
return new RepositoryData(genId, newSnapshotIds, indexSnapshots, incompatibleSnapshotIds);
}
/**
* Returns a new {@link RepositoryData} instance containing the same snapshot data as the
* invoking instance, with the given incompatible snapshots added to the new instance.
*/
public RepositoryData addIncompatibleSnapshots(final List<SnapshotId> incompatibleSnapshotIds) {
List<SnapshotId> newSnapshotIds = new ArrayList<>(this.snapshotIds);
List<SnapshotId> newIncompatibleSnapshotIds = new ArrayList<>(this.incompatibleSnapshotIds);
for (SnapshotId snapshotId : incompatibleSnapshotIds) {
newSnapshotIds.remove(snapshotId);
newIncompatibleSnapshotIds.add(snapshotId);
}
return new RepositoryData(this.genId, newSnapshotIds, this.indexSnapshots, newIncompatibleSnapshotIds);
return new RepositoryData(genId, newSnapshotIds, newSnapshotStates, indexSnapshots, incompatibleSnapshotIds);
}
/**
@ -219,7 +230,7 @@ public final class RepositoryData {
* Initializes the indices in the repository metadata; returns a new instance.
*/
public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
return new RepositoryData(genId, snapshotIds, snapshotStates, indexSnapshots, incompatibleSnapshotIds);
}
@Override
@ -232,6 +243,7 @@ public final class RepositoryData {
}
@SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj;
return snapshotIds.equals(that.snapshotIds)
&& snapshotStates.equals(that.snapshotStates)
&& indices.equals(that.indices)
&& indexSnapshots.equals(that.indexSnapshots)
&& incompatibleSnapshotIds.equals(that.incompatibleSnapshotIds);
@ -239,7 +251,7 @@ public final class RepositoryData {
@Override
public int hashCode() {
return Objects.hash(snapshotIds, indices, indexSnapshots, incompatibleSnapshotIds);
return Objects.hash(snapshotIds, snapshotStates, indices, indexSnapshots, incompatibleSnapshotIds);
}
/**
@ -291,6 +303,9 @@ public final class RepositoryData {
private static final String INCOMPATIBLE_SNAPSHOTS = "incompatible-snapshots";
private static final String INDICES = "indices";
private static final String INDEX_ID = "id";
private static final String NAME = "name";
private static final String UUID = "uuid";
private static final String STATE = "state";
/**
* Writes the snapshots metadata and the related indices metadata to x-content, omitting the
@ -301,7 +316,13 @@ public final class RepositoryData {
// write the snapshots list
builder.startArray(SNAPSHOTS);
for (final SnapshotId snapshot : getSnapshotIds()) {
snapshot.toXContent(builder, params);
builder.startObject();
builder.field(NAME, snapshot.getName());
builder.field(UUID, snapshot.getUUID());
if (snapshotStates.containsKey(snapshot.getUUID())) {
builder.field(STATE, snapshotStates.get(snapshot.getUUID()).value());
}
builder.endObject();
}
builder.endArray();
// write the indices map
@ -313,7 +334,7 @@ public final class RepositoryData {
Set<SnapshotId> snapshotIds = indexSnapshots.get(indexId);
assert snapshotIds != null;
for (final SnapshotId snapshotId : snapshotIds) {
snapshotId.toXContent(builder, params);
builder.value(snapshotId.getUUID());
}
builder.endArray();
builder.endObject();
@ -327,20 +348,47 @@ public final class RepositoryData {
* Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
*/
public static RepositoryData snapshotsFromXContent(final XContentParser parser, long genId) throws IOException {
List<SnapshotId> snapshots = new ArrayList<>();
Map<String, SnapshotId> snapshots = new HashMap<>();
Map<String, SnapshotState> snapshotStates = new HashMap<>();
Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
if (SNAPSHOTS.equals(currentFieldName)) {
String field = parser.currentName();
if (SNAPSHOTS.equals(field)) {
if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
snapshots.add(SnapshotId.fromXContent(parser));
final SnapshotId snapshotId;
// the new format from 5.0 which contains the snapshot name and uuid
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
String name = null;
String uuid = null;
SnapshotState state = null;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
String currentFieldName = parser.currentName();
parser.nextToken();
if (NAME.equals(currentFieldName)) {
name = parser.text();
} else if (UUID.equals(currentFieldName)) {
uuid = parser.text();
} else if (STATE.equals(currentFieldName)) {
state = SnapshotState.fromValue(parser.numberValue().byteValue());
}
}
snapshotId = new SnapshotId(name, uuid);
if (state != null) {
snapshotStates.put(uuid, state);
}
} else {
// the old format pre 5.0 that only contains the snapshot name, use the name as the uuid too
final String name = parser.text();
snapshotId = new SnapshotId(name, name);
}
snapshots.put(snapshotId.getUUID(), snapshotId);
}
} else {
throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]");
throw new ElasticsearchParseException("expected array for [" + field + "]");
}
} else if (INDICES.equals(currentFieldName)) {
} else if (INDICES.equals(field)) {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticsearchParseException("start object expected [indices]");
}
@ -361,7 +409,22 @@ public final class RepositoryData {
throw new ElasticsearchParseException("start array expected [snapshots]");
}
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
snapshotIds.add(SnapshotId.fromXContent(parser));
String uuid = null;
// the old format pre 5.4.1 which contains the snapshot name and uuid
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
String currentFieldName = parser.currentName();
parser.nextToken();
if (UUID.equals(currentFieldName)) {
uuid = parser.text();
}
}
} else {
// the new format post 5.4.1 that only contains the snapshot uuid,
// since we already have the name/uuid combo in the snapshots array
uuid = parser.text();
}
snapshotIds.add(snapshots.get(uuid));
}
}
}
@ -369,13 +432,13 @@ public final class RepositoryData {
indexSnapshots.put(new IndexId(indexName, indexId), snapshotIds);
}
} else {
throw new ElasticsearchParseException("unknown field name [" + currentFieldName + "]");
throw new ElasticsearchParseException("unknown field name [" + field + "]");
}
}
} else {
throw new ElasticsearchParseException("start object expected");
}
return new RepositoryData(genId, snapshots, indexSnapshots, Collections.emptyList());
return new RepositoryData(genId, snapshots, snapshotStates, indexSnapshots, Collections.emptyList());
}
/**
@ -419,7 +482,7 @@ public final class RepositoryData {
} else {
throw new ElasticsearchParseException("start object expected");
}
return new RepositoryData(this.genId, this.snapshotIds, this.indexSnapshots, incompatibleSnapshotIds);
return new RepositoryData(this.genId, this.snapshotIds, this.snapshotStates, this.indexSnapshots, incompatibleSnapshotIds);
}
}

5
core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -473,10 +473,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
shardFailures);
snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
final RepositoryData repositoryData = getRepositoryData();
List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
if (!snapshotIds.contains(snapshotId)) {
writeIndexGen(repositoryData.addSnapshot(snapshotId, indices), repositoryStateId);
}
writeIndexGen(repositoryData.addSnapshot(snapshotId, blobStoreSnapshot.state(), indices), repositoryStateId);
return blobStoreSnapshot;
} catch (IOException ex) {
throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);

3
core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
View File

@ -42,7 +42,6 @@ public class RestGetSnapshotsAction extends BaseRestHandler {
controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}", this);
}
@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
String repository = request.param("repository");
@ -50,7 +49,7 @@ public class RestGetSnapshotsAction extends BaseRestHandler {
GetSnapshotsRequest getSnapshotsRequest = getSnapshotsRequest(repository).snapshots(snapshots);
getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable()));
getSnapshotsRequest.verbose(request.paramAsBoolean("verbose", getSnapshotsRequest.verbose()));
getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout()));
return channel -> client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestToXContentListener<>(channel));
}

7
core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
View File

@ -32,7 +32,7 @@ import java.util.Objects;
/**
* SnapshotId - snapshot name + snapshot UUID
*/
public final class SnapshotId implements Writeable, ToXContent {
public final class SnapshotId implements Comparable<SnapshotId>, Writeable, ToXContent {
private static final String NAME = "name";
private static final String UUID = "uuid";
@ -106,6 +106,11 @@ public final class SnapshotId implements Writeable, ToXContent {
return hashCode;
}
@Override
public int compareTo(final SnapshotId other) {
return this.name.compareTo(other.name);
}
private int computeHashCode() {
return Objects.hash(name, uuid);
}

114
core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
@ -69,11 +70,17 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
private static final String SUCCESSFUL_SHARDS = "successful_shards";
private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0_UNRELEASED;
public static final Version VERBOSE_INTRODUCED = Version.V_6_0_0_alpha1_UNRELEASED;
private static final Comparator<SnapshotInfo> COMPARATOR =
Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId);
private final SnapshotId snapshotId;
@Nullable
private final SnapshotState state;
@Nullable
private final String reason;
private final List<String> indices;
@ -91,6 +98,10 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
private final List<SnapshotShardFailure> shardFailures;
public SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state) {
this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, Collections.emptyList());
}
public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime) {
this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
}
@ -104,8 +115,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
private SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state, String reason, Version version,
long startTime, long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures) {
this.snapshotId = Objects.requireNonNull(snapshotId);
this.indices = Objects.requireNonNull(indices);
this.state = Objects.requireNonNull(state);
this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices));
this.state = state;
this.reason = reason;
this.version = version;
this.startTime = startTime;
@ -126,7 +137,11 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
indicesListBuilder.add(in.readString());
}
indices = Collections.unmodifiableList(indicesListBuilder);
state = SnapshotState.fromValue(in.readByte());
if (in.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
state = in.readBoolean() ? SnapshotState.fromValue(in.readByte()) : null;
} else {
state = SnapshotState.fromValue(in.readByte());
}
reason = in.readOptionalString();
startTime = in.readVLong();
endTime = in.readVLong();
@ -159,6 +174,14 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
null, 0L, 0L, 0, 0, Collections.emptyList());
}
/**
* Gets a new {@link SnapshotInfo} instance from the given {@link SnapshotInfo} with
* all information stripped out except the snapshot id, state, and indices.
*/
public SnapshotInfo basic() {
return new SnapshotInfo(snapshotId, indices, state);
}
/**
* Returns snapshot id
*
@ -169,25 +192,27 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns snapshot state
* Returns snapshot state; {@code null} if the state is unknown.
*
* @return snapshot state
*/
@Nullable
public SnapshotState state() {
return state;
}
/**
* Returns snapshot failure reason
* Returns snapshot failure reason; {@code null} if the snapshot succeeded.
*
* @return snapshot failure reason
*/
@Nullable
public String reason() {
return reason;
}
/**
* Returns indices that were included into this snapshot
* Returns indices that were included in this snapshot.
*
* @return list of indices
*/
@ -196,7 +221,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns time when snapshot started
* Returns time when snapshot started; a value of {@code 0L} will be returned if
* {@link #state()} returns {@code null}.
*
* @return snapshot start time
*/
@ -205,9 +231,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns time when snapshot ended
* <p>
* Can be 0L if snapshot is still running
* Returns time when snapshot ended; a value of {@code 0L} will be returned if the
* snapshot is still running or if {@link #state()} returns {@code null}.
*
* @return snapshot end time
*/
@ -216,7 +241,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns total number of shards that were snapshotted
* Returns total number of shards that were snapshotted; a value of {@code 0} will
* be returned if {@link #state()} returns {@code null}.
*
* @return number of shards
*/
@ -225,7 +251,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Number of failed shards
* Number of failed shards; a value of {@code 0} will be returned if there were no
* failed shards, or if {@link #state()} returns {@code null}.
*
* @return number of failed shards
*/
@ -234,7 +261,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns total number of shards that were successfully snapshotted
* Returns total number of shards that were successfully snapshotted; a value of
* {@code 0} will be returned if {@link #state()} returns {@code null}.
*
* @return number of successful shards
*/
@ -243,7 +271,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Returns shard failures
* Returns shard failures; an empty list will be returned if there were no shard
* failures, or if {@link #state()} returns {@code null}.
*
* @return shard failures
*/
@ -253,7 +282,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
/**
* Returns the version of elasticsearch that the snapshot was created with. Will only
* return {@code null} if {@link #state()} returns {@link SnapshotState#INCOMPATIBLE}.
* return {@code null} if {@link #state()} returns {@code null} or {@link SnapshotState#INCOMPATIBLE}.
*
* @return version of elasticsearch that the snapshot was created with
*/
@ -263,16 +292,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
}
/**
* Compares two snapshots by their start time
*
* @param o other snapshot
* @return the value {@code 0} if snapshots were created at the same time;
* a value less than {@code 0} if this snapshot was created before snapshot {@code o}; and
* a value greater than {@code 0} if this snapshot was created after snapshot {@code o};
* Compares two snapshots by their start time; if the start times are the same, then
* compares the two snapshots by their snapshot ids.
*/
@Override
public int compareTo(final SnapshotInfo o) {
return Long.compare(startTime, o.startTime);
return COMPARATOR.compare(this, o);
}
@Override
@ -328,15 +353,15 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
if (version != null) {
builder.field(VERSION_ID, version.id);
builder.field(VERSION, version.toString());
} else {
builder.field(VERSION, "unknown");
}
builder.startArray(INDICES);
for (String index : indices) {
builder.value(index);
}
builder.endArray();
builder.field(STATE, state);
if (state != null) {
builder.field(STATE, state);
}
if (reason != null) {
builder.field(REASON, reason);
}
@ -349,18 +374,22 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
builder.field(END_TIME_IN_MILLIS, endTime);
builder.timeValueField(DURATION_IN_MILLIS, DURATION, endTime - startTime);
}
builder.startArray(FAILURES);
for (SnapshotShardFailure shardFailure : shardFailures) {
builder.startObject();
shardFailure.toXContent(builder, params);
if (!shardFailures.isEmpty()) {
builder.startArray(FAILURES);
for (SnapshotShardFailure shardFailure : shardFailures) {
builder.startObject();
shardFailure.toXContent(builder, params);
builder.endObject();
}
builder.endArray();
}
if (totalShards != 0) {
builder.startObject(SHARDS);
builder.field(TOTAL, totalShards);
builder.field(FAILED, failedShards());
builder.field(SUCCESSFUL, successfulShards);
builder.endObject();
}
builder.endArray();
builder.startObject(SHARDS);
builder.field(TOTAL, totalShards);
builder.field(FAILED, failedShards());
builder.field(SUCCESSFUL, successfulShards);
builder.endObject();
builder.endObject();
return builder;
}
@ -496,10 +525,19 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
for (String index : indices) {
out.writeString(index);
}
if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
out.writeByte(SnapshotState.FAILED.value());
if (out.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
if (state != null) {
out.writeBoolean(true);
out.writeByte(state.value());
} else {
out.writeBoolean(false);
}
} else {
out.writeByte(state.value());
if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
out.writeByte(SnapshotState.FAILED.value());
} else {
out.writeByte(state.value());
}
}
out.writeOptionalString(reason);
out.writeVLong(startTime);

5
core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
View File

@ -26,8 +26,6 @@ import org.apache.logging.log4j.util.Supplier;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.OriginalIndices;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
@ -67,7 +65,6 @@ import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
@ -171,7 +168,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
*/
public List<SnapshotInfo> snapshots(final String repositoryName,
final List<SnapshotId> snapshotIds,
final List<SnapshotId> incompatibleSnapshotIds,
final Set<SnapshotId> incompatibleSnapshotIds,
final boolean ignoreUnavailable) {
final Set<SnapshotInfo> snapshotSet = new HashSet<>();
final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);

8
core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
View File

@ -161,6 +161,14 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
assertThat(snapshotInfo.version().toString(), equalTo(version));
logger.info("--> get less verbose snapshot info");
getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo)
.setSnapshots(snapshot).setVerbose(false).get();
assertEquals(1, getSnapshotsResponse.getSnapshots().size());
snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
assertEquals(snapshot, snapshotInfo.snapshotId().getName());
assertNull(snapshotInfo.version()); // in verbose=false mode, version doesn't exist
logger.info("--> restoring snapshot");
RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get();
assertThat(response.status(), equalTo(RestStatus.OK));

2
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -1538,7 +1538,7 @@ public class IndexShardTests extends IndexShardTestCase {
public RepositoryData getRepositoryData() {
Map<IndexId, Set<SnapshotId>> map = new HashMap<>();
map.put(new IndexId(indexName, "blah"), emptySet());
return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), map, Collections.emptyList());
return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), map, Collections.emptyList());
}
@Override

55
core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
@ -82,7 +83,8 @@ public class RepositoryDataTests extends ESTestCase {
for (int i = 0; i < numOld; i++) {
indices.add(indexIdMap.get(indexNames.get(i)));
}
RepositoryData newRepoData = repositoryData.addSnapshot(newSnapshot, indices);
RepositoryData newRepoData = repositoryData.addSnapshot(newSnapshot,
randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indices);
// verify that the new repository data has the new snapshot and its indices
assertTrue(newRepoData.getSnapshotIds().contains(newSnapshot));
for (IndexId indexId : indices) {
@ -97,15 +99,21 @@ public class RepositoryDataTests extends ESTestCase {
public void testInitIndices() {
final int numSnapshots = randomIntBetween(1, 30);
final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
final Map<String, SnapshotId> snapshotIds = new HashMap<>(numSnapshots);
for (int i = 0; i < numSnapshots; i++) {
snapshotIds.add(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
snapshotIds.put(snapshotId.getUUID(), snapshotId);
}
RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), Collections.emptyList());
RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds,
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList());
// test that initializing indices works
Map<IndexId, Set<SnapshotId>> indices = randomIndices(snapshotIds);
RepositoryData newRepoData = repositoryData.initIndices(indices);
assertEquals(repositoryData.getSnapshotIds(), newRepoData.getSnapshotIds());
List<SnapshotId> expected = new ArrayList<>(repositoryData.getSnapshotIds());
Collections.sort(expected);
List<SnapshotId> actual = new ArrayList<>(newRepoData.getSnapshotIds());
Collections.sort(actual);
assertEquals(expected, actual);
for (IndexId indexId : indices.keySet()) {
assertEquals(indices.get(indexId), newRepoData.getSnapshots(indexId));
}
@ -136,25 +144,32 @@ public class RepositoryDataTests extends ESTestCase {
assertEquals(new IndexId(notInRepoData, notInRepoData), repositoryData.resolveIndexId(notInRepoData));
}
public void testGetSnapshotState() {
final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
final SnapshotState state = randomFrom(SnapshotState.values());
final RepositoryData repositoryData = RepositoryData.EMPTY.addSnapshot(snapshotId, state, Collections.emptyList());
assertEquals(state, repositoryData.getSnapshotState(snapshotId));
assertNull(repositoryData.getSnapshotState(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID())));
}
public static RepositoryData generateRandomRepoData() {
return generateRandomRepoData(new ArrayList<>());
}
public static RepositoryData generateRandomRepoData(final List<SnapshotId> origSnapshotIds) {
List<SnapshotId> snapshotIds = randomSnapshots(origSnapshotIds);
return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, randomIndices(snapshotIds), Collections.emptyList());
}
private static List<SnapshotId> randomSnapshots(final List<SnapshotId> origSnapshotIds) {
final int numSnapshots = randomIntBetween(1, 30);
final List<SnapshotId> snapshotIds = new ArrayList<>(origSnapshotIds);
for (int i = 0; i < numSnapshots; i++) {
snapshotIds.add(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
final int numIndices = randomIntBetween(1, 30);
final List<IndexId> indices = new ArrayList<>(numIndices);
for (int i = 0; i < numIndices; i++) {
indices.add(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
}
return snapshotIds;
final int numSnapshots = randomIntBetween(1, 30);
RepositoryData repositoryData = RepositoryData.EMPTY;
for (int i = 0; i < numSnapshots; i++) {
final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
final List<IndexId> someIndices = indices.subList(0, randomIntBetween(1, numIndices));
repositoryData = repositoryData.addSnapshot(snapshotId, randomFrom(SnapshotState.values()), someIndices);
}
return repositoryData;
}
private static Map<IndexId, Set<SnapshotId>> randomIndices(final List<SnapshotId> snapshotIds) {
private static Map<IndexId, Set<SnapshotId>> randomIndices(final Map<String, SnapshotId> snapshotIdsMap) {
final List<SnapshotId> snapshotIds = new ArrayList<>(snapshotIdsMap.values());
final int totalSnapshots = snapshotIds.size();
final int numIndices = randomIntBetween(1, 30);
final Map<IndexId, Set<SnapshotId>> indices = new HashMap<>(numIndices);

10
core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESSingleNodeTestCase;
@ -143,7 +144,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L));
// removing a snapshot and writing to a new index generational file
repositoryData = repository.getRepositoryData().removeSnapshot(repositoryData.getSnapshotIds().get(0));
repositoryData = repository.getRepositoryData().removeSnapshot(repositoryData.getSnapshotIds().iterator().next());
repository.writeIndexGen(repositoryData, repositoryData.getGenId());
assertEquals(repository.getRepositoryData(), repositoryData);
assertThat(repository.latestIndexBlobId(), equalTo(2L));
@ -181,8 +182,8 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
for (int i = 0; i < numSnapshots; i++) {
snapshotIds.add(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
}
RepositoryData repositoryData = new RepositoryData(readData.getGenId(), Collections.emptyList(), Collections.emptyMap(),
snapshotIds);
RepositoryData repositoryData = new RepositoryData(readData.getGenId(),
Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), snapshotIds);
repository.blobContainer().deleteBlob("incompatible-snapshots");
repository.writeIncompatibleSnapshots(repositoryData);
readData = repository.getRepositoryData();
@ -228,7 +229,8 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
for (int j = 0; j < numIndices; j++) {
indexIds.add(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()));
}
repoData = repoData.addSnapshot(snapshotId, indexIds);
repoData = repoData.addSnapshot(snapshotId,
randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indexIds);
}
return repoData;
}

89
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -88,8 +88,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@ -2754,4 +2756,91 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
}
}
}
public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
logger.info("--> creating repository");
final Path repoPath = randomRepoPath();
final Client client = client();
assertAcked(client.admin().cluster()
.preparePutRepository("test-repo")
.setType("fs")
.setVerify(false)
.setSettings(Settings.builder().put("location", repoPath)));
logger.info("--> creating random number of indices");
final int numIndices = randomIntBetween(1, 10);
for (int i = 0; i < numIndices; i++) {
assertAcked(prepareCreate("test-idx-" + i).setSettings(Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)));
}
logger.info("--> creating random number of snapshots");
final int numSnapshots = randomIntBetween(1, 10);
final Map<String, List<String>> indicesPerSnapshot = new HashMap<>();
for (int i = 0; i < numSnapshots; i++) {
// index some additional docs (maybe) for each index
for (int j = 0; j < numIndices; j++) {
if (randomBoolean()) {
final int numDocs = randomIntBetween(1, 5);
for (int k = 0; k < numDocs; k++) {
index("test-idx-" + j, "doc", Integer.toString(k), "foo", "bar" + k);
}
refresh();
}
}
final boolean all = randomBoolean();
boolean atLeastOne = false;
List<String> indices = new ArrayList<>();
for (int j = 0; j < numIndices; j++) {
if (all || randomBoolean() || !atLeastOne) {
indices.add("test-idx-" + j);
atLeastOne = true;
}
}
final String snapshotName = "test-snap-" + i;
indicesPerSnapshot.put(snapshotName, indices);
client.admin().cluster()
.prepareCreateSnapshot("test-repo", snapshotName)
.setWaitForCompletion(true)
.setIndices(indices.toArray(new String[indices.size()]))
.get();
}
logger.info("--> verify _all returns snapshot info");
GetSnapshotsResponse response = client().admin().cluster()
.prepareGetSnapshots("test-repo")
.setSnapshots("_all")
.setVerbose(false)
.get();
assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
verifySnapshotInfo(response, indicesPerSnapshot);
logger.info("--> verify wildcard returns snapshot info");
response = client().admin().cluster()
.prepareGetSnapshots("test-repo")
.setSnapshots("test-snap-*")
.setVerbose(false)
.get();
assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
verifySnapshotInfo(response, indicesPerSnapshot);
logger.info("--> verify individual requests return snapshot info");
for (int i = 0; i < numSnapshots; i++) {
response = client().admin().cluster()
.prepareGetSnapshots("test-repo")
.setSnapshots("test-snap-" + i)
.setVerbose(false)
.get();
assertEquals(1, response.getSnapshots().size());
verifySnapshotInfo(response, indicesPerSnapshot);
}
}
private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
final List<String> expected = snapshotInfo.indices();
assertEquals(expected, indicesPerSnapshot.get(snapshotInfo.snapshotId().getName()));
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
}
}
}

9
docs/reference/modules/snapshots.asciidoc
View File

@ -350,6 +350,15 @@ GET /_snapshot/my_backup/_all
The command fails if some of the snapshots are unavailable. The boolean parameter `ignore_unavailable` can be used to
return all snapshots that are currently available.
Getting all snapshots in the repository can be costly on cloud-based repositories,
both from a cost and performance perspective. If the only information required is
the snapshot names/uuids in the repository and the indices in each snapshot, then
the optional boolean parameter `verbose` can be set to `false` to execute a more
performant and cost-effective retrieval of the snapshots in the repository. Note
that setting `verbose` to `false` will omit all other information about the snapshot
such as status information, the number of snapshotted shards, etc. The default
value of the `verbose` parameter is `true`.
A currently running snapshot can be retrieved using the following command:
[source,sh]

4
rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
View File

@ -25,6 +25,10 @@
"ignore_unavailable": {
"type": "boolean",
"description": "Whether to ignore unavailable snapshots, defaults to false which means a SnapshotMissingException is thrown"
},
"verbose": {
"type": "boolean",
"description": "Whether to show verbose snapshot info or only show the basic info found in the repository index blob"
}
}
},

41
rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yaml
View File

@ -33,6 +33,11 @@ setup:
- is_true: snapshots
- do:
snapshot.delete:
repository: test_repo_get_1
snapshot: test_snapshot
---
"Get missing snapshot info throws an exception":
@ -52,3 +57,39 @@ setup:
ignore_unavailable: true
- is_true: snapshots
---
"Get snapshot info when verbose is false":
- skip:
version: " - 5.99.99"
reason: verbose mode was introduced in 6.0
- do:
indices.create:
index: test_index
body:
settings:
number_of_shards: 1
number_of_replicas: 0
- do:
snapshot.create:
repository: test_repo_get_1
snapshot: test_snapshot
wait_for_completion: true
- do:
snapshot.get:
repository: test_repo_get_1
snapshot: test_snapshot
verbose: false
- is_true: snapshots
- match: { snapshots.0.snapshot: test_snapshot }
- match: { snapshots.0.state: SUCCESS }
- is_false: snapshots.0.version
- do:
snapshot.delete:
repository: test_repo_get_1
snapshot: test_snapshot