honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Snapshot/Restore: Allow custom metadata to specify whether or not it should be in a snapshot 

Before this change all persistent custom metadata is stored as part of snapshot. It requires us to remove repositories metadata later during recovery process. This change allows custom metadata to specify whether or not it should be stored as part of a snapshot.

  Fixes #7900
This commit is contained in:
Igor Motov 2014-09-26 23:03:04 +04:00
parent ddbeb910be
commit b7a4c6da65
9 changed files with 443 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.action.admin.cluster.state;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
@ -30,12 +32,18 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaData.Custom;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.util.List;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.cluster.metadata.MetaData.lookupFactorySafe;
/**
*
*/
@ -118,8 +126,18 @@ public class TransportClusterStateAction extends TransportMasterNodeReadOperatio
}
}
// Filter our metadata that shouldn't be returned by API
for(ObjectCursor<String> type : currentState.metaData().customs().keys()) {
Custom.Factory factory = lookupFactorySafe(type.value);
if(!factory.context().contains(MetaData.XContentContext.API)) {
mdBuilder.removeCustom(type.value);
}
}
builder.metaData(mdBuilder);
}
listener.onResponse(new ClusterStateResponse(clusterName, builder.build()));
}
}

6
src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java
View File

@ -155,7 +155,7 @@ public class BenchmarkMetaData implements MetaData.Custom {
}
public static class Factory implements MetaData.Custom.Factory<BenchmarkMetaData> {
public static class Factory extends MetaData.Custom.Factory<BenchmarkMetaData> {
@Override
public String type() {
@ -209,10 +209,6 @@ public class BenchmarkMetaData implements MetaData.Custom {
builder.endArray();
builder.endObject();
}
public boolean isPersistent() {
return false;
}
}
public boolean contains(String benchmarkId) {

55
src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View File

@ -60,24 +60,38 @@ public class MetaData implements Iterable<IndexMetaData> {
public static final String ALL = "_all";
public enum XContentContext {
/* Custom metadata should be returns as part of API call */
API,
/* Custom metadata should be stored as part of the persistent cluster state */
GATEWAY,
/* Custom metadata should be stored as part of a snapshot */
SNAPSHOT;
}
public static EnumSet<XContentContext> API_ONLY = EnumSet.of(XContentContext.API);
public static EnumSet<XContentContext> API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
public static EnumSet<XContentContext> API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
public interface Custom {
interface Factory<T extends Custom> {
abstract class Factory<T extends Custom> {
String type();
public abstract String type();
T readFrom(StreamInput in) throws IOException;
public abstract T readFrom(StreamInput in) throws IOException;
void writeTo(T customIndexMetaData, StreamOutput out) throws IOException;
public abstract void writeTo(T customIndexMetaData, StreamOutput out) throws IOException;
T fromXContent(XContentParser parser) throws IOException;
public abstract T fromXContent(XContentParser parser) throws IOException;
void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException;
public abstract void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException;
/**
* Returns true if this custom metadata should be persisted as part of global cluster state
*/
boolean isPersistent();
public EnumSet<XContentContext> context() {
return API_ONLY;
}
}
}
@ -118,9 +132,11 @@ public class MetaData implements Iterable<IndexMetaData> {
public static final MetaData EMPTY_META_DATA = builder().build();
public static final String GLOBAL_ONLY_PARAM = "global_only";
public static final String CONTEXT_MODE_PARAM = "context_mode";
public static final String PERSISTENT_ONLY_PARAM = "persistent_only";
public static final String CONTEXT_MODE_SNAPSHOT = XContentContext.SNAPSHOT.toString();
public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();
private final String uuid;
private final long version;
@ -1076,14 +1092,14 @@ public class MetaData implements Iterable<IndexMetaData> {
// Check if any persistent metadata needs to be saved
int customCount1 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData1.customs) {
if (customFactories.get(cursor.key).isPersistent()) {
if (customFactories.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
if (!cursor.value.equals(metaData2.custom(cursor.key))) return false;
customCount1++;
}
}
int customCount2 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData2.customs) {
if (customFactories.get(cursor.key).isPersistent()) {
if (customFactories.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
customCount2++;
}
}
@ -1262,8 +1278,8 @@ public class MetaData implements Iterable<IndexMetaData> {
}
public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
boolean globalOnly = params.paramAsBoolean(GLOBAL_ONLY_PARAM, false);
boolean persistentOnly = params.paramAsBoolean(PERSISTENT_ONLY_PARAM, false);
XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));
builder.startObject("meta-data");
builder.field("version", metaData.version());
@ -1277,7 +1293,7 @@ public class MetaData implements Iterable<IndexMetaData> {
builder.endObject();
}
if (!persistentOnly && !metaData.transientSettings().getAsMap().isEmpty()) {
if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) {
builder.startObject("transient_settings");
for (Map.Entry<String, String> entry : metaData.transientSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
@ -1291,7 +1307,7 @@ public class MetaData implements Iterable<IndexMetaData> {
}
builder.endObject();
if (!globalOnly && !metaData.indices().isEmpty()) {
if (context == XContentContext.API && !metaData.indices().isEmpty()) {
builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData) {
IndexMetaData.Builder.toXContent(indexMetaData, builder, params);
@ -1301,13 +1317,12 @@ public class MetaData implements Iterable<IndexMetaData> {
for (ObjectObjectCursor<String, Custom> cursor : metaData.customs()) {
Custom.Factory factory = lookupFactorySafe(cursor.key);
if (!persistentOnly || factory.isPersistent()) {
if(factory.context().contains(context)) {
builder.startObject(cursor.key);
factory.toXContent(cursor.value, builder, params);
builder.endObject();
}
}
builder.endObject();
}

17
src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@ -82,7 +83,7 @@ public class RepositoriesMetaData implements MetaData.Custom {
/**
* Repository metadata factory
*/
public static class Factory implements MetaData.Custom.Factory<RepositoriesMetaData> {
public static class Factory extends MetaData.Custom.Factory<RepositoriesMetaData> {
/**
* {@inheritDoc}
@ -171,6 +172,11 @@ public class RepositoriesMetaData implements MetaData.Custom {
}
}
@Override
public EnumSet<MetaData.XContentContext> context() {
return MetaData.API_AND_GATEWAY;
}
/**
* Serializes information about a single repository
*
@ -190,15 +196,6 @@ public class RepositoriesMetaData implements MetaData.Custom {
builder.endObject();
}
/**
* {@inheritDoc}
*/
@Override
public boolean isPersistent() {
return true;
}
}
}

11
src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java
View File

@ -397,7 +397,7 @@ public class RestoreMetaData implements MetaData.Custom {
/**
* Restore metadata factory
*/
public static class Factory implements MetaData.Custom.Factory<RestoreMetaData> {
public static class Factory extends MetaData.Custom.Factory<RestoreMetaData> {
/**
* {@inheritDoc}
@ -512,15 +512,6 @@ public class RestoreMetaData implements MetaData.Custom {
builder.endArray();
builder.endObject();
}
/**
* {@inheritDoc}
*/
@Override
public boolean isPersistent() {
return false;
}
}

7
src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java
View File

@ -312,7 +312,7 @@ public class SnapshotMetaData implements MetaData.Custom {
}
public static class Factory implements MetaData.Custom.Factory<SnapshotMetaData> {
public static class Factory extends MetaData.Custom.Factory<SnapshotMetaData> {
@Override
public String type() {
@ -410,11 +410,6 @@ public class SnapshotMetaData implements MetaData.Custom {
builder.endArray();
builder.endObject();
}
public boolean isPersistent() {
return false;
}
}

20
src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java
View File

@ -106,7 +106,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
private final XContentType format;
private final ToXContent.Params formatParams;
private final ToXContent.Params globalOnlyFormatParams;
private final ToXContent.Params gatewayModeFormatParams;
private final AutoImportDangledState autoImportDangled;
@ -130,17 +130,15 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
Map<String, String> params = Maps.newHashMap();
params.put("binary", "true");
formatParams = new ToXContent.MapParams(params);
Map<String, String> globalOnlyParams = Maps.newHashMap();
globalOnlyParams.put("binary", "true");
globalOnlyParams.put(MetaData.PERSISTENT_ONLY_PARAM, "true");
globalOnlyParams.put(MetaData.GLOBAL_ONLY_PARAM, "true");
globalOnlyFormatParams = new ToXContent.MapParams(globalOnlyParams);
Map<String, String> gatewayModeParams = Maps.newHashMap();
gatewayModeParams.put("binary", "true");
gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
} else {
formatParams = ToXContent.EMPTY_PARAMS;
Map<String, String> globalOnlyParams = Maps.newHashMap();
globalOnlyParams.put(MetaData.PERSISTENT_ONLY_PARAM, "true");
globalOnlyParams.put(MetaData.GLOBAL_ONLY_PARAM, "true");
globalOnlyFormatParams = new ToXContent.MapParams(globalOnlyParams);
Map<String, String> gatewayModeParams = Maps.newHashMap();
gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
}
this.autoImportDangled = AutoImportDangledState.fromString(settings.get("gateway.local.auto_import_dangled", AutoImportDangledState.YES.toString()));
@ -399,7 +397,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
XContentBuilder builder = XContentFactory.contentBuilder(format);
builder.startObject();
MetaData.Builder.toXContent(metaData, builder, globalOnlyFormatParams);
MetaData.Builder.toXContent(metaData, builder, gatewayModeFormatParams);
builder.endObject();
builder.flush();
String globalFileName = "global-" + metaData.version();

13
src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -120,7 +120,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
private final BlobStoreIndexShardRepository indexShardRepository;
private final ToXContent.Params globalOnlyFormatParams;
private final ToXContent.Params snapshotOnlyFormatParams;
private final RateLimiter snapshotRateLimiter;
@ -142,10 +142,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
super(repositorySettings.globalSettings());
this.repositoryName = repositoryName;
this.indexShardRepository = (BlobStoreIndexShardRepository) indexShardRepository;
Map<String, String> globalOnlyParams = Maps.newHashMap();
globalOnlyParams.put(MetaData.PERSISTENT_ONLY_PARAM, "true");
globalOnlyParams.put(MetaData.GLOBAL_ONLY_PARAM, "true");
globalOnlyFormatParams = new ToXContent.MapParams(globalOnlyParams);
Map<String, String> snpashotOnlyParams = Maps.newHashMap();
snpashotOnlyParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_SNAPSHOT);
snapshotOnlyFormatParams = new ToXContent.MapParams(snpashotOnlyParams);
snapshotRateLimiter = getRateLimiter(repositorySettings, "max_snapshot_bytes_per_sec", new ByteSizeValue(20, ByteSizeUnit.MB));
restoreRateLimiter = getRateLimiter(repositorySettings, "max_restore_bytes_per_sec", new ByteSizeValue(20, ByteSizeUnit.MB));
}
@ -555,7 +554,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
}
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
builder.startObject();
BlobStoreSnapshot.Builder.toXContent(snapshot, builder, globalOnlyFormatParams);
BlobStoreSnapshot.Builder.toXContent(snapshot, builder, snapshotOnlyFormatParams);
builder.endObject();
builder.close();
}
@ -574,7 +573,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
}
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
builder.startObject();
MetaData.Builder.toXContent(metaData, builder, globalOnlyFormatParams);
MetaData.Builder.toXContent(metaData, builder, snapshotOnlyFormatParams);
builder.endObject();
builder.close();
}

367
src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java
View File

@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
@ -33,21 +33,35 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.store.MockDirectoryHelper;
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
@ -97,6 +111,135 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
.getMetaData().persistentSettings().get(ThreadPool.THREADPOOL_GROUP + "dummy.value"), equalTo(settingValue));
}
@Test
public void restoreCustomMetadata() throws Exception {
File tempDir = newTempDir();
logger.info("--> start node");
internalCluster().startNode(settingsBuilder().put("gateway.type", "local"));
Client client = client();
createIndex("test-idx");
ensureYellow();
logger.info("--> add custom persistent metadata");
updateClusterState(new ClusterStateUpdater() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
ClusterState.Builder builder = ClusterState.builder(currentState);
MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s"));
metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns"));
metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw"));
metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw"));
metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi"));
builder.metaData(metadataBuilder);
return builder.build();
}
});
logger.info("--> create repository");
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
logger.info("--> start snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
logger.info("--> change custom persistent metadata");
updateClusterState(new ClusterStateUpdater() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
ClusterState.Builder builder = ClusterState.builder(currentState);
MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
if (randomBoolean()) {
metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s"));
} else {
metadataBuilder.removeCustom(SnapshottableMetadata.TYPE);
}
metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns"));
if (randomBoolean()) {
metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw"));
} else {
metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE);
}
metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw"));
metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE);
builder.metaData(metadataBuilder);
return builder.build();
}
});
logger.info("--> delete repository");
assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo"));
logger.info("--> create repository");
putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2")
.setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
logger.info("--> restore snapshot");
client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet();
logger.info("--> make sure old repository wasn't restored");
assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class);
assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1));
logger.info("--> check that custom persistent metadata was restored");
ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
logger.info("Cluster state: {}", clusterState);
MetaData metaData = clusterState.getMetaData();
assertThat(((SnapshottableMetadata)metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s"));
assertThat(((NonSnapshottableMetadata)metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns"));
assertThat(((SnapshottableGatewayMetadata)metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw"));
assertThat(((NonSnapshottableGatewayMetadata)metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw"));
logger.info("--> restart all nodes");
internalCluster().fullRestart();
ensureYellow();
logger.info("--> check that gateway-persistent custom metadata survived full cluster restart");
clusterState = client().admin().cluster().prepareState().get().getState();
logger.info("Cluster state: {}", clusterState);
metaData = clusterState.getMetaData();
assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue());
assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue());
assertThat(((SnapshottableGatewayMetadata)metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw"));
assertThat(((NonSnapshottableGatewayMetadata)metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw"));
// Shouldn't be returned as part of API response
assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue());
// But should still be in state
metaData = internalCluster().getInstance(ClusterService.class).state().metaData();
assertThat(((SnapshotableGatewayNoApiMetadata)metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi"));
}
private void updateClusterState(final ClusterStateUpdater updater) throws InterruptedException {
final CountDownLatch countDownLatch = new CountDownLatch(1);
final ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
clusterService.submitStateUpdateTask("test", new ProcessedClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
return updater.execute(currentState);
}
@Override
public void onFailure(String source, @Nullable Throwable t) {
countDownLatch.countDown();
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
countDownLatch.countDown();
}
});
countDownLatch.await();
}
private static interface ClusterStateUpdater {
public ClusterState execute(ClusterState currentState) throws Exception;
}
@Test
public void snapshotDuringNodeShutdownTest() throws Exception {
logger.info("--> start 2 nodes");
@ -484,4 +627,224 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000))
));
}
public static abstract class TestCustomMetaData implements MetaData.Custom {
private final String data;
protected TestCustomMetaData(String data) {
this.data = data;
}
public String getData() {
return data;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TestCustomMetaData that = (TestCustomMetaData) o;
if (!data.equals(that.data)) return false;
return true;
}
@Override
public int hashCode() {
return data.hashCode();
}
public static abstract class TestCustomMetaDataFactory<T extends TestCustomMetaData> extends MetaData.Custom.Factory<T> {
protected abstract TestCustomMetaData newTestCustomMetaData(String data);
@Override
public T readFrom(StreamInput in) throws IOException {
return (T)newTestCustomMetaData(in.readString());
}
@Override
public void writeTo(T metadata, StreamOutput out) throws IOException {
out.writeString(metadata.getData());
}
@Override
public T fromXContent(XContentParser parser) throws IOException {
XContentParser.Token token;
String data = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
if ("data".equals(currentFieldName)) {
if (parser.nextToken() != XContentParser.Token.VALUE_STRING) {
throw new ElasticsearchParseException("failed to parse snapshottable metadata, invalid data type");
}
data = parser.text();
} else {
throw new ElasticsearchParseException("failed to parse snapshottable metadata, unknown field [" + currentFieldName + "]");
}
} else {
throw new ElasticsearchParseException("failed to parse snapshottable metadata");
}
}
if (data == null) {
throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found");
}
return (T)newTestCustomMetaData(data);
}
@Override
public void toXContent(T metadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.field("data", metadata.getData());
}
}
}
static {
MetaData.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY);
MetaData.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY);
MetaData.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY);
MetaData.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY);
MetaData.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY);
}
public static class SnapshottableMetadata extends TestCustomMetaData {
public static final String TYPE = "test_snapshottable";
public static final Factory FACTORY = new Factory();
public SnapshottableMetadata(String data) {
super(data);
}
private static class Factory extends TestCustomMetaDataFactory<SnapshottableMetadata> {
@Override
public String type() {
return TYPE;
}
@Override
protected TestCustomMetaData newTestCustomMetaData(String data) {
return new SnapshottableMetadata(data);
}
@Override
public EnumSet<MetaData.XContentContext> context() {
return MetaData.API_AND_SNAPSHOT;
}
}
}
public static class NonSnapshottableMetadata extends TestCustomMetaData {
public static final String TYPE = "test_non_snapshottable";
public static final Factory FACTORY = new Factory();
public NonSnapshottableMetadata(String data) {
super(data);
}
private static class Factory extends TestCustomMetaDataFactory<NonSnapshottableMetadata> {
@Override
public String type() {
return TYPE;
}
@Override
protected NonSnapshottableMetadata newTestCustomMetaData(String data) {
return new NonSnapshottableMetadata(data);
}
}
}
public static class SnapshottableGatewayMetadata extends TestCustomMetaData {
public static final String TYPE = "test_snapshottable_gateway";
public static final Factory FACTORY = new Factory();
public SnapshottableGatewayMetadata(String data) {
super(data);
}
private static class Factory extends TestCustomMetaDataFactory<SnapshottableGatewayMetadata> {
@Override
public String type() {
return TYPE;
}
@Override
protected TestCustomMetaData newTestCustomMetaData(String data) {
return new SnapshottableGatewayMetadata(data);
}
@Override
public EnumSet<MetaData.XContentContext> context() {
return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.SNAPSHOT, MetaData.XContentContext.GATEWAY);
}
}
}
public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData {
public static final String TYPE = "test_non_snapshottable_gateway";
public static final Factory FACTORY = new Factory();
public NonSnapshottableGatewayMetadata(String data) {
super(data);
}
private static class Factory extends TestCustomMetaDataFactory<NonSnapshottableGatewayMetadata> {
@Override
public String type() {
return TYPE;
}
@Override
protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) {
return new NonSnapshottableGatewayMetadata(data);
}
@Override
public EnumSet<MetaData.XContentContext> context() {
return MetaData.API_AND_GATEWAY;
}
}
}
public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData {
public static final String TYPE = "test_snapshottable_gateway_no_api";
public static final Factory FACTORY = new Factory();
public SnapshotableGatewayNoApiMetadata(String data) {
super(data);
}
private static class Factory extends TestCustomMetaDataFactory<SnapshotableGatewayNoApiMetadata> {
@Override
public String type() {
return TYPE;
}
@Override
protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) {
return new SnapshotableGatewayNoApiMetadata(data);
}
@Override
public EnumSet<MetaData.XContentContext> context() {
return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT);
}
}
}
}