Standardize state format type for global and index level metadata

Currently, global and index level state format type can be configured through gateway.format.
This commit removes the ability to configure format type for these states.
Now we always store these states in SMILE format and ensure we always write them
to disk in the most compact way.
This commit is contained in:
Areek Zillur 2016-03-15 17:43:48 -04:00
parent 0a12e7bb5b
commit 7aeeb52cf6
7 changed files with 88 additions and 144 deletions

View File

@ -46,6 +46,8 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
@ -215,6 +217,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
.numberOfShards(1).numberOfReplicas(0).build(); .numberOfShards(1).numberOfReplicas(0).build();
public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations"; public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations";
public static final String INDEX_STATE_FILE_PREFIX = "state-";
private final int numberOfShards; private final int numberOfShards;
private final int numberOfReplicas; private final int numberOfReplicas;
@ -1023,4 +1026,21 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
return builder.build(); return builder.build();
} }
private static final ToXContent.Params FORMAT_PARAMS = new MapParams(Collections.singletonMap("binary", "true"));
/**
* State format for {@link IndexMetaData} to write to and load from disk
*/
public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, INDEX_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
Builder.toXContent(state, builder, FORMAT_PARAMS);
}
@Override
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
return Builder.fromXContent(parser);
}
};
} }

View File

@ -51,6 +51,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.store.IndexStoreConfig; import org.elasticsearch.index.store.IndexStoreConfig;
@ -153,6 +154,8 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString(); public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();
public static final String GLOBAL_STATE_FILE_PREFIX = "global-";
private final String clusterUUID; private final String clusterUUID;
private final long version; private final long version;
@ -1160,4 +1163,28 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
return PROTO.readFrom(in); return PROTO.readFrom(in);
} }
} }
private final static ToXContent.Params FORMAT_PARAMS;
static {
Map<String, String> params = new HashMap<>(2);
params.put("binary", "true");
params.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
FORMAT_PARAMS = new MapParams(params);
}
/**
* State format for {@link MetaData} to write to and load from disk
*/
public final static MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
Builder.toXContent(state, builder, FORMAT_PARAMS);
}
@Override
public MetaData fromXContent(XContentParser parser) throws IOException {
return Builder.fromXContent(parser);
}
};
} }

View File

@ -47,7 +47,6 @@ public class IndexFolderUpgrader {
private final NodeEnvironment nodeEnv; private final NodeEnvironment nodeEnv;
private final Settings settings; private final Settings settings;
private final ESLogger logger = Loggers.getLogger(IndexFolderUpgrader.class); private final ESLogger logger = Loggers.getLogger(IndexFolderUpgrader.class);
private final MetaDataStateFormat<IndexMetaData> indexStateFormat = readOnlyIndexMetaDataStateFormat();
/** /**
* Creates a new upgrader instance * Creates a new upgrader instance
@ -90,7 +89,7 @@ public class IndexFolderUpgrader {
void upgrade(final String indexFolderName) throws IOException { void upgrade(final String indexFolderName) throws IOException {
for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) { for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName); final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
final IndexMetaData indexMetaData = indexStateFormat.loadLatestState(logger, indexFolderPath); final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
if (indexMetaData != null) { if (indexMetaData != null) {
final Index index = indexMetaData.getIndex(); final Index index = indexMetaData.getIndex();
if (needsUpgrade(index, indexFolderName)) { if (needsUpgrade(index, indexFolderName)) {
@ -135,20 +134,4 @@ public class IndexFolderUpgrader {
static boolean needsUpgrade(Index index, String indexFolderName) { static boolean needsUpgrade(Index index, String indexFolderName) {
return indexFolderName.equals(index.getUUID()) == false; return indexFolderName.equals(index.getUUID()) == false;
} }
static MetaDataStateFormat<IndexMetaData> readOnlyIndexMetaDataStateFormat() {
// NOTE: XContentType param is not used as we use the format read from the serialized index state
return new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, MetaStateService.INDEX_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
return IndexMetaData.Builder.fromXContent(parser);
}
};
}
} }

View File

@ -25,19 +25,13 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.function.Predicate; import java.util.function.Predicate;
/** /**
@ -45,41 +39,12 @@ import java.util.function.Predicate;
*/ */
public class MetaStateService extends AbstractComponent { public class MetaStateService extends AbstractComponent {
static final String FORMAT_SETTING = "gateway.format";
static final String GLOBAL_STATE_FILE_PREFIX = "global-";
public static final String INDEX_STATE_FILE_PREFIX = "state-";
private final NodeEnvironment nodeEnv; private final NodeEnvironment nodeEnv;
private final XContentType format;
private final ToXContent.Params formatParams;
private final ToXContent.Params gatewayModeFormatParams;
private final MetaDataStateFormat<IndexMetaData> indexStateFormat;
private final MetaDataStateFormat<MetaData> globalStateFormat;
@Inject @Inject
public MetaStateService(Settings settings, NodeEnvironment nodeEnv) { public MetaStateService(Settings settings, NodeEnvironment nodeEnv) {
super(settings); super(settings);
this.nodeEnv = nodeEnv; this.nodeEnv = nodeEnv;
this.format = XContentType.fromMediaTypeOrFormat(settings.get(FORMAT_SETTING, "smile"));
if (this.format == XContentType.SMILE) {
Map<String, String> params = new HashMap<>();
params.put("binary", "true");
formatParams = new ToXContent.MapParams(params);
Map<String, String> gatewayModeParams = new HashMap<>();
gatewayModeParams.put("binary", "true");
gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
} else {
formatParams = ToXContent.EMPTY_PARAMS;
Map<String, String> gatewayModeParams = new HashMap<>();
gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
}
indexStateFormat = indexStateFormat(format, formatParams);
globalStateFormat = globalStateFormat(format, gatewayModeFormatParams);
} }
/** /**
@ -95,7 +60,7 @@ public class MetaStateService extends AbstractComponent {
metaDataBuilder = MetaData.builder(); metaDataBuilder = MetaData.builder();
} }
for (String indexFolderName : nodeEnv.availableIndexFolders()) { for (String indexFolderName : nodeEnv.availableIndexFolders()) {
IndexMetaData indexMetaData = indexStateFormat.loadLatestState(logger, nodeEnv.resolveIndexFolder(indexFolderName)); IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.resolveIndexFolder(indexFolderName));
if (indexMetaData != null) { if (indexMetaData != null) {
metaDataBuilder.put(indexMetaData, false); metaDataBuilder.put(indexMetaData, false);
} else { } else {
@ -110,7 +75,7 @@ public class MetaStateService extends AbstractComponent {
*/ */
@Nullable @Nullable
IndexMetaData loadIndexState(Index index) throws IOException { IndexMetaData loadIndexState(Index index) throws IOException {
return indexStateFormat.loadLatestState(logger, nodeEnv.indexPaths(index)); return IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
} }
/** /**
@ -122,7 +87,7 @@ public class MetaStateService extends AbstractComponent {
if (excludeIndexPathIdsPredicate.test(indexFolderName)) { if (excludeIndexPathIdsPredicate.test(indexFolderName)) {
continue; continue;
} }
IndexMetaData indexMetaData = indexStateFormat.loadLatestState(logger, IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
nodeEnv.resolveIndexFolder(indexFolderName)); nodeEnv.resolveIndexFolder(indexFolderName));
if (indexMetaData != null) { if (indexMetaData != null) {
final String indexPathId = indexMetaData.getIndex().getUUID(); final String indexPathId = indexMetaData.getIndex().getUUID();
@ -142,7 +107,7 @@ public class MetaStateService extends AbstractComponent {
* Loads the global state, *without* index state, see {@link #loadFullState()} for that. * Loads the global state, *without* index state, see {@link #loadFullState()} for that.
*/ */
MetaData loadGlobalState() throws IOException { MetaData loadGlobalState() throws IOException {
MetaData globalState = globalStateFormat.loadLatestState(logger, nodeEnv.nodeDataPaths()); MetaData globalState = MetaData.FORMAT.loadLatestState(logger, nodeEnv.nodeDataPaths());
// ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing // ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing
// TODO: can we somehow only do this for pre-2.0 cluster state? // TODO: can we somehow only do this for pre-2.0 cluster state?
if (globalState != null) { if (globalState != null) {
@ -167,7 +132,7 @@ public class MetaStateService extends AbstractComponent {
final Index index = indexMetaData.getIndex(); final Index index = indexMetaData.getIndex();
logger.trace("[{}] writing state, reason [{}]", index, reason); logger.trace("[{}] writing state, reason [{}]", index, reason);
try { try {
indexStateFormat.write(indexMetaData, indexMetaData.getVersion(), locations); IndexMetaData.FORMAT.write(indexMetaData, indexMetaData.getVersion(), locations);
} catch (Throwable ex) { } catch (Throwable ex) {
logger.warn("[{}]: failed to write index state", ex, index); logger.warn("[{}]: failed to write index state", ex, index);
throw new IOException("failed to write state for [" + index + "]", ex); throw new IOException("failed to write state for [" + index + "]", ex);
@ -180,45 +145,10 @@ public class MetaStateService extends AbstractComponent {
void writeGlobalState(String reason, MetaData metaData) throws Exception { void writeGlobalState(String reason, MetaData metaData) throws Exception {
logger.trace("[_global] writing state, reason [{}]", reason); logger.trace("[_global] writing state, reason [{}]", reason);
try { try {
globalStateFormat.write(metaData, metaData.version(), nodeEnv.nodeDataPaths()); MetaData.FORMAT.write(metaData, metaData.version(), nodeEnv.nodeDataPaths());
} catch (Throwable ex) { } catch (Throwable ex) {
logger.warn("[_global]: failed to write global state", ex); logger.warn("[_global]: failed to write global state", ex);
throw new IOException("failed to write global state", ex); throw new IOException("failed to write global state", ex);
} }
} }
/**
* Returns a StateFormat that can read and write {@link MetaData}
*/
static MetaDataStateFormat<MetaData> globalStateFormat(XContentType format, final ToXContent.Params formatParams) {
return new MetaDataStateFormat<MetaData>(format, GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
MetaData.Builder.toXContent(state, builder, formatParams);
}
@Override
public MetaData fromXContent(XContentParser parser) throws IOException {
return MetaData.Builder.fromXContent(parser);
}
};
}
/**
* Returns a StateFormat that can read and write {@link IndexMetaData}
*/
static MetaDataStateFormat<IndexMetaData> indexStateFormat(XContentType format, final ToXContent.Params formatParams) {
return new MetaDataStateFormat<IndexMetaData>(format, INDEX_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
IndexMetaData.Builder.toXContent(state, builder, formatParams); }
@Override
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
return IndexMetaData.Builder.fromXContent(parser);
}
};
}
} }

View File

@ -30,14 +30,9 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.MetaDataStateFormat; import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.gateway.MetaStateService;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardId;
@ -63,25 +58,9 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.hamcrest.core.Is.is;
@LuceneTestCase.SuppressFileSystems("ExtrasFS") @LuceneTestCase.SuppressFileSystems("ExtrasFS")
public class IndexFolderUpgraderTests extends ESTestCase { public class IndexFolderUpgraderTests extends ESTestCase {
private static MetaDataStateFormat<IndexMetaData> indexMetaDataStateFormat =
new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, MetaStateService.INDEX_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
IndexMetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
}
@Override
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
return IndexMetaData.Builder.fromXContent(parser);
}
};
/** /**
* tests custom data paths are upgraded * tests custom data paths are upgraded
*/ */
@ -244,7 +223,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
assertEquals(indexFolders.size(), 1); assertEquals(indexFolders.size(), 1);
// ensure index metadata is moved // ensure index metadata is moved
IndexMetaData indexMetaData = indexMetaDataStateFormat.loadLatestState(logger, IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next())); nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
assertNotNull(indexMetaData); assertNotNull(indexMetaData);
Index index = indexMetaData.getIndex(); Index index = indexMetaData.getIndex();
@ -277,7 +256,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
.numberOfReplicas(0) .numberOfReplicas(0)
.build(); .build();
try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
indexMetaDataStateFormat.write(indexState, 1, nodeEnvironment.indexPaths(index)); IndexMetaData.FORMAT.write(indexState, 1, nodeEnvironment.indexPaths(index));
assertFalse(IndexFolderUpgrader.needsUpgrade(index, index.getUUID())); assertFalse(IndexFolderUpgrader.needsUpgrade(index, index.getUUID()));
} }
} }
@ -286,7 +265,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
int numIdxFiles, int numTranslogFiles) throws IOException { int numIdxFiles, int numTranslogFiles) throws IOException {
final Index index = indexSettings.getIndex(); final Index index = indexSettings.getIndex();
// ensure index state can be loaded // ensure index state can be loaded
IndexMetaData loadLatestState = indexMetaDataStateFormat.loadLatestState(logger, nodeEnv.indexPaths(index)); IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
assertNotNull(loadLatestState); assertNotNull(loadLatestState);
assertEquals(loadLatestState.getIndex(), index); assertEquals(loadLatestState.getIndex(), index);
for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) { for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) {
@ -326,7 +305,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
for (int i = 0; i < nodePaths.length; i++) { for (int i = 0; i < nodePaths.length; i++) {
oldIndexPaths[i] = nodePaths[i].indicesPath.resolve(indexSettings.getIndex().getName()); oldIndexPaths[i] = nodePaths[i].indicesPath.resolve(indexSettings.getIndex().getName());
} }
indexMetaDataStateFormat.write(indexSettings.getIndexMetaData(), 1, oldIndexPaths); IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), 1, oldIndexPaths);
for (int id = 0; id < indexSettings.getNumberOfShards(); id++) { for (int id = 0; id < indexSettings.getNumberOfShards(); id++) {
Path oldIndexPath = randomFrom(oldIndexPaths); Path oldIndexPath = randomFrom(oldIndexPaths);
ShardId shardId = new ShardId(indexSettings.getIndex(), id); ShardId shardId = new ShardId(indexSettings.getIndex(), id);

View File

@ -234,8 +234,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
// If the latest version doesn't use the legacy format while previous versions do, then fail hard // If the latest version doesn't use the legacy format while previous versions do, then fail hard
public void testLatestVersionDoesNotUseLegacy() throws IOException { public void testLatestVersionDoesNotUseLegacy() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS; MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params);
final Path[] dirs = new Path[2]; final Path[] dirs = new Path[2];
dirs[0] = createTempDir(); dirs[0] = createTempDir();
dirs[1] = createTempDir(); dirs[1] = createTempDir();
@ -252,9 +251,10 @@ public class MetaDataStateFormatTests extends ESTestCase {
for (int i = 0; i < numLegacyFiles; ++i) { for (int i = 0; i < numLegacyFiles; ++i) {
final Path dir2 = randomFrom(dirs); final Path dir2 = randomFrom(dirs);
final int v2 = v1 + 1 + randomInt(10); final int v2 = v1 + 1 + randomInt(10);
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaStateService.GLOBAL_STATE_FILE_PREFIX + v2)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(),
Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaData.GLOBAL_STATE_FILE_PREFIX + v2)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); format.toXContent(xcontentBuilder, randomMeta());
xcontentBuilder.endObject(); xcontentBuilder.endObject();
} }
} }
@ -279,8 +279,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
// If both the legacy and the new format are available for the latest version, prefer the new format // If both the legacy and the new format are available for the latest version, prefer the new format
public void testPrefersNewerFormat() throws IOException { public void testPrefersNewerFormat() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS; MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params);
final Path[] dirs = new Path[2]; final Path[] dirs = new Path[2];
dirs[0] = createTempDir(); dirs[0] = createTempDir();
dirs[1] = createTempDir(); dirs[1] = createTempDir();
@ -296,9 +295,10 @@ public class MetaDataStateFormatTests extends ESTestCase {
final Path dir2 = randomFrom(dirs); final Path dir2 = randomFrom(dirs);
MetaData meta2 = randomMeta(); MetaData meta2 = randomMeta();
assertFalse(meta2.clusterUUID().equals(uuid)); assertFalse(meta2.clusterUUID().equals(uuid));
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaStateService.GLOBAL_STATE_FILE_PREFIX + v)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaData.FORMAT.format(),
Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaData.GLOBAL_STATE_FILE_PREFIX + v)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); format.toXContent(xcontentBuilder, randomMeta());
xcontentBuilder.endObject(); xcontentBuilder.endObject();
} }
@ -312,7 +312,6 @@ public class MetaDataStateFormatTests extends ESTestCase {
} }
public void testLoadState() throws IOException { public void testLoadState() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
final Path[] dirs = new Path[randomIntBetween(1, 5)]; final Path[] dirs = new Path[randomIntBetween(1, 5)];
int numStates = randomIntBetween(1, 5); int numStates = randomIntBetween(1, 5);
int numLegacy = randomIntBetween(0, numStates); int numLegacy = randomIntBetween(0, numStates);
@ -321,7 +320,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
meta.add(randomMeta()); meta.add(randomMeta());
} }
Set<Path> corruptedFiles = new HashSet<>(); Set<Path> corruptedFiles = new HashSet<>();
MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params); MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
dirs[i] = createTempDir(); dirs[i] = createTempDir();
Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
@ -331,9 +330,10 @@ public class MetaDataStateFormatTests extends ESTestCase {
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j); Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
} else { } else {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaData.FORMAT.format(),
Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params); format.toXContent(xcontentBuilder, meta.get(j));
xcontentBuilder.endObject(); xcontentBuilder.endObject();
} }
} }
@ -380,7 +380,20 @@ public class MetaDataStateFormatTests extends ESTestCase {
assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue()); assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
} }
} }
}
private static MetaDataStateFormat<MetaData> metaDataFormat(XContentType format) {
return new MetaDataStateFormat<MetaData>(format, MetaData.GLOBAL_STATE_FILE_PREFIX) {
@Override
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
}
@Override
public MetaData fromXContent(XContentParser parser) throws IOException {
return MetaData.Builder.fromXContent(parser);
}
};
} }
private MetaData randomMeta() throws IOException { private MetaData randomMeta() throws IOException {

View File

@ -41,7 +41,7 @@ public class MetaStateServiceTests extends ESTestCase {
public void testWriteLoadIndex() throws Exception { public void testWriteLoadIndex() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) { try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env); MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build(); IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
metaStateService.writeIndex("test_write", index); metaStateService.writeIndex("test_write", index);
@ -51,14 +51,14 @@ public class MetaStateServiceTests extends ESTestCase {
public void testLoadMissingIndex() throws Exception { public void testLoadMissingIndex() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) { try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env); MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue()); assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
} }
} }
public void testWriteLoadGlobal() throws Exception { public void testWriteLoadGlobal() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) { try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env); MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
MetaData metaData = MetaData.builder() MetaData metaData = MetaData.builder()
.persistentSettings(Settings.builder().put("test1", "value1").build()) .persistentSettings(Settings.builder().put("test1", "value1").build())
@ -70,7 +70,7 @@ public class MetaStateServiceTests extends ESTestCase {
public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception { public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) { try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env); MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
MetaData metaData = MetaData.builder() MetaData metaData = MetaData.builder()
.persistentSettings(Settings.builder().put("test1", "value1").build()) .persistentSettings(Settings.builder().put("test1", "value1").build())
@ -86,7 +86,7 @@ public class MetaStateServiceTests extends ESTestCase {
public void testLoadGlobal() throws Exception { public void testLoadGlobal() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) { try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env); MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build(); IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
MetaData metaData = MetaData.builder() MetaData metaData = MetaData.builder()
@ -103,12 +103,4 @@ public class MetaStateServiceTests extends ESTestCase {
assertThat(loadedState.index("test1"), equalTo(index)); assertThat(loadedState.index("test1"), equalTo(index));
} }
} }
private Settings randomSettings() {
Settings.Builder builder = Settings.builder();
if (randomBoolean()) {
builder.put(MetaStateService.FORMAT_SETTING, randomFrom(XContentType.values()).shortName());
}
return builder.build();
}
} }