From a16abf921fa9795f7b869dae3977373bcfa02260 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 10 Dec 2019 09:45:27 +0100 Subject: [PATCH] Make elasticsearch-node tools custom metadata-aware (#48390) The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently unaware of plugins and will therefore drop custom metadata from the cluster state once the state is written out again (as it skips over the custom metadata that it can't read). This commit preserves unknown customs when editing on-disk metadata through the elasticsearch-node command-line tools. --- .../testclusters/ElasticsearchNode.java | 18 ++--- .../common/xcontent/XContentBuilder.java | 19 +++-- .../cli/EnvironmentAwareCommand.java | 13 ++- .../ElasticsearchNodeCommand.java | 9 +-- .../UnsafeBootstrapMasterCommand.java | 3 +- .../cluster/metadata/IndexMetaData.java | 3 - .../cluster/metadata/MetaData.java | 81 +++++++++++++++---- .../env/NodeRepurposeCommand.java | 5 +- .../env/OverrideNodeVersionCommand.java | 3 +- .../cluster/metadata/MetaDataTests.java | 2 +- .../metadata/ToAndFromJsonMetaDataTests.java | 2 +- .../gateway/MetaDataStateFormatTests.java | 51 ++++++++++-- .../LicensesMetaDataSerializationTests.java | 2 +- .../WatcherMetaDataSerializationTests.java | 2 +- 14 files changed, 156 insertions(+), 57 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index a06b0448384..8c16bf6cf37 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -418,7 +418,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { if (plugins.isEmpty() == false) { logToProcessStdout("Installing " + plugins.size() + " plugins"); - plugins.forEach(plugin -> runElaticsearchBinScript( + plugins.forEach(plugin -> runElasticsearchBinScript( "elasticsearch-plugin", "install", "--batch", plugin.toString()) ); @@ -426,7 +426,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) { LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion()); - runElaticsearchBinScript( + runElasticsearchBinScript( "elasticsearch-plugin", "install", "--batch", "x-pack" ); @@ -434,10 +434,10 @@ public class ElasticsearchNode implements TestClusterConfiguration { if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) { logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files"); - runElaticsearchBinScript("elasticsearch-keystore", "create"); + runElasticsearchBinScript("elasticsearch-keystore", "create"); keystoreSettings.forEach((key, value) -> - runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key) + runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key) ); for (Map.Entry entry : keystoreFiles.entrySet()) { @@ -446,7 +446,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { if (file.exists() == false) { throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this); } - runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath()); + runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath()); } } @@ -463,7 +463,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { if (credentials.isEmpty() == false) { logToProcessStdout("Setting up " + credentials.size() + " users"); - credentials.forEach(paramMap -> runElaticsearchBinScript( + credentials.forEach(paramMap -> runElasticsearchBinScript( getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users", paramMap.entrySet().stream() .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) @@ -592,7 +592,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { credentials.add(cred); } - private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) { + private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) { if ( Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false && Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false @@ -632,8 +632,8 @@ public class ElasticsearchNode implements TestClusterConfiguration { } } - private void runElaticsearchBinScript(String tool, String... args) { - runElaticsearchBinScriptWithInput("", tool, args); + private void runElasticsearchBinScript(String tool, String... args) { + runElasticsearchBinScriptWithInput("", tool, args); } private Map getESEnvironment() { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 51a4f86a0d3..20fde0891b6 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -819,7 +819,7 @@ public final class XContentBuilder implements Closeable, Flushable { } else if (value instanceof Map) { @SuppressWarnings("unchecked") final Map valueMap = (Map) value; - map(valueMap, ensureNoSelfReferences); + map(valueMap, ensureNoSelfReferences, true); } else if (value instanceof Iterable) { value((Iterable) value, ensureNoSelfReferences); } else if (value instanceof Object[]) { @@ -867,10 +867,15 @@ public final class XContentBuilder implements Closeable, Flushable { } public XContentBuilder map(Map values) throws IOException { - return map(values, true); + return map(values, true, true); } - private XContentBuilder map(Map values, boolean ensureNoSelfReferences) throws IOException { + /** writes a map without the start object and end object headers */ + public XContentBuilder mapContents(Map values) throws IOException { + return map(values, true, false); + } + + private XContentBuilder map(Map values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException { if (values == null) { return nullValue(); } @@ -881,13 +886,17 @@ public final class XContentBuilder implements Closeable, Flushable { ensureNoSelfReferences(values); } - startObject(); + if (writeStartAndEndHeaders) { + startObject(); + } for (Map.Entry value : values.entrySet()) { field(value.getKey()); // pass ensureNoSelfReferences=false as we already performed the check at a higher level unknownValue(value.getValue(), false); } - endObject(); + if (writeStartAndEndHeaders) { + endObject(); + } return this; } diff --git a/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java b/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java index 6fc3349c762..1d3a31f0a72 100644 --- a/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java +++ b/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java @@ -88,14 +88,19 @@ public abstract class EnvironmentAwareCommand extends Command { /** Create an {@link Environment} for the command to use. Overrideable for tests. */ protected Environment createEnv(final Map settings) throws UserException { + return createEnv(Settings.EMPTY, settings); + } + + /** Create an {@link Environment} for the command to use. Overrideable for tests. */ + protected final Environment createEnv(final Settings baseSettings, final Map settings) throws UserException { final String esPathConf = System.getProperty("es.path.conf"); if (esPathConf == null) { throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set"); } - return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings, - getConfigPath(esPathConf), - // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available - () -> System.getenv("HOSTNAME")); + return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings, + getConfigPath(esPathConf), + // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available + () -> System.getenv("HOSTNAME")); } @SuppressForbidden(reason = "need path to construct environment") diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java index fbfcc4672bb..00e3eb2e241 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -27,7 +27,6 @@ import org.apache.lucene.store.LockObtainFailedException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cli.EnvironmentAwareCommand; import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.Tuple; @@ -43,7 +42,6 @@ import java.util.Objects; public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class); - protected final NamedXContentRegistry namedXContentRegistry; protected static final String DELIMITER = "------------------------------------------------------------------------\n"; static final String STOP_WARNING_MSG = @@ -65,7 +63,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { super(description); nodeOrdinalOption = parser.accepts("ordinal", "Optional node ordinal, 0 if not specified") .withRequiredArg().ofType(Integer.class); - namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); } protected void processNodePathsWithLock(Terminal terminal, OptionSet options, Environment env) throws IOException { @@ -88,7 +85,7 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { protected Tuple loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file"); - final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (manifest == null) { throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG); @@ -97,8 +94,8 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG); } terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file"); - final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), - dataPaths); + final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration( + logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths); if (metaData == null) { throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index c15e832142e..05bc0116c13 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeMetaData; import org.elasticsearch.node.Node; @@ -84,7 +85,7 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand { protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata"); - final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (nodeMetaData == null) { throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b1c80dbf880..461c719986e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -45,7 +45,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -1513,8 +1512,6 @@ public class IndexMetaData implements Diffable, ToXContentFragmen @Override public IndexMetaData fromXContent(XContentParser parser) throws IOException { - assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY - : "loading index metadata requires a working named xcontent registry"; return Builder.fromXContent(parser); } }; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 16ee761af6e..32cb8e572a0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -813,7 +813,7 @@ public class MetaData implements Iterable, Diffable, To } public static MetaData fromXContent(XContentParser parser) throws IOException { - return Builder.fromXContent(parser); + return Builder.fromXContent(parser, false); } @Override @@ -1353,7 +1353,7 @@ public class MetaData implements Iterable, Diffable, To builder.endObject(); } - public static MetaData fromXContent(XContentParser parser) throws IOException { + public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException { Builder builder = new Builder(); // we might get here after the meta-data element, or on a fresh parser @@ -1403,8 +1403,13 @@ public class MetaData implements Iterable, Diffable, To Custom custom = parser.namedObject(Custom.class, currentFieldName, null); builder.putCustom(custom.getWriteableName(), custom); } catch (NamedObjectNotFoundException ex) { - logger.warn("Skipping unknown custom object with type {}", currentFieldName); - parser.skipChildren(); + if (preserveUnknownCustoms) { + logger.warn("Adding unknown custom object with type {}", currentFieldName); + builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered())); + } else { + logger.warn("Skipping unknown custom object with type {}", currentFieldName); + parser.skipChildren(); + } } } } else if (token.isValue()) { @@ -1425,6 +1430,45 @@ public class MetaData implements Iterable, Diffable, To } } + public static class UnknownGatewayOnlyCustom implements Custom { + + private final Map contents; + + UnknownGatewayOnlyCustom(Map contents) { + this.contents = contents; + } + + @Override + public EnumSet context() { + return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY); + } + + @Override + public Diff diff(Custom previousState) { + throw new UnsupportedOperationException(); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException(); + } + + @Override + public Version getMinimalSupportedVersion() { + throw new UnsupportedOperationException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.mapContents(contents); + } + } + private static final ToXContent.Params FORMAT_PARAMS; static { Map params = new HashMap<>(2); @@ -1436,16 +1480,25 @@ public class MetaData implements Iterable, Diffable, To /** * State format for {@link MetaData} to write to and load from disk */ - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = createMetaDataStateFormat(false); - @Override - public void toXContent(XContentBuilder builder, MetaData state) throws IOException { - Builder.toXContent(state, builder, FORMAT_PARAMS); - } + /** + * Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs + */ + public static final MetaDataStateFormat FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true); - @Override - public MetaData fromXContent(XContentParser parser) throws IOException { - return Builder.fromXContent(parser); - } - }; + private static MetaDataStateFormat createMetaDataStateFormat(boolean preserveUnknownCustoms) { + return new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { + + @Override + public void toXContent(XContentBuilder builder, MetaData state) throws IOException { + Builder.toXContent(state, builder, FORMAT_PARAMS); + } + + @Override + public MetaData fromXContent(XContentParser parser) throws IOException { + return Builder.fromXContent(parser, preserveUnknownCustoms); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java b/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java index cfe3ab6b3fd..f3e099f8938 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java +++ b/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.WriteStateException; @@ -165,7 +166,7 @@ public class NodeRepurposeCommand extends ElasticsearchNodeCommand { indexPaths[i] = nodePaths[i].resolve(uuid); } try { - IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths); + IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths); return metaData.getIndex().getName(); } catch (Exception e) { return "no name for uuid: " + uuid + ": " + e; @@ -194,7 +195,7 @@ public class NodeRepurposeCommand extends ElasticsearchNodeCommand { private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest"); - final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (manifest == null) { terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE); diff --git a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java index 34c7e9599e0..f50bdf081ef 100644 --- a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java +++ b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java @@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import java.io.IOException; import java.nio.file.Path; @@ -74,7 +75,7 @@ public class OverrideNodeVersionCommand extends ElasticsearchNodeCommand { protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new); final NodeMetaData nodeMetaData - = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths); + = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths); if (nodeMetaData == null) { throw new ElasticsearchException(NO_METADATA_MESSAGE); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index e9893f16f95..47050d8bd10 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -365,7 +365,7 @@ public class MetaDataTests extends ESTestCase { .endObject() .endObject()); try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) { - MetaData.Builder.fromXContent(parser); + MetaData.Builder.fromXContent(parser, randomBoolean()); fail(); } catch (IllegalArgumentException e) { assertEquals("Unexpected field [random]", e.getMessage()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index ae24915e32d..805445da1aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -147,7 +147,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase { String metaDataSource = MetaData.Builder.toXContent(metaData); - MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource)); + MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false); IndexMetaData indexMetaData = parsedMetaData.index("test1"); assertThat(indexMetaData.primaryTerm(0), equalTo(1L)); diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 40f3bd8a016..c7dab0dc4d4 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -61,6 +61,8 @@ import java.util.Set; import java.util.stream.StreamSupport; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -80,7 +82,7 @@ public class MetaDataStateFormatTests extends ESTestCase { @Override public MetaData fromXContent(XContentParser parser) throws IOException { - return MetaData.Builder.fromXContent(parser); + return MetaData.Builder.fromXContent(parser, false); } }; Path tmp = createTempDir(); @@ -233,7 +235,23 @@ public class MetaDataStateFormatTests extends ESTestCase { } } - public void testLoadState() throws IOException { + public void testLoadStateWithoutMissingCustoms() throws IOException { + runLoadStateTest(false, false); + } + + public void testLoadStateWithoutMissingCustomsButPreserved() throws IOException { + runLoadStateTest(false, true); + } + + public void testLoadStateWithMissingCustomsButPreserved() throws IOException { + runLoadStateTest(true, true); + } + + public void testLoadStateWithMissingCustomsAndNotPreserved() throws IOException { + runLoadStateTest(true, false); + } + + private void runLoadStateTest(boolean hasMissingCustoms, boolean preserveUnknownCustoms) throws IOException { final Path[] dirs = new Path[randomIntBetween(1, 5)]; int numStates = randomIntBetween(1, 5); List meta = new ArrayList<>(); @@ -241,7 +259,7 @@ public class MetaDataStateFormatTests extends ESTestCase { meta.add(randomMeta()); } Set corruptedFiles = new HashSet<>(); - MetaDataStateFormat format = metaDataFormat(); + MetaDataStateFormat format = metaDataFormat(preserveUnknownCustoms); for (int i = 0; i < dirs.length; i++) { dirs[i] = createTempDir(); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); @@ -258,11 +276,12 @@ public class MetaDataStateFormatTests extends ESTestCase { } List dirList = Arrays.asList(dirs); Collections.shuffle(dirList, random()); - MetaData loadedMetaData = format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0])); + MetaData loadedMetaData = format.loadLatestState(logger, hasMissingCustoms ? + NamedXContentRegistry.EMPTY : xContentRegistry(), dirList.toArray(new Path[0])); MetaData latestMetaData = meta.get(numStates-1); assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_"))); assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID())); - ImmutableOpenMap indices = loadedMetaData.indices(); + ImmutableOpenMap indices = loadedMetaData.indices(); assertThat(indices.size(), equalTo(latestMetaData.indices().size())); for (IndexMetaData original : latestMetaData) { IndexMetaData deserialized = indices.get(original.getIndex().getName()); @@ -275,7 +294,23 @@ public class MetaDataStateFormatTests extends ESTestCase { } // make sure the index tombstones are the same too - assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + if (hasMissingCustoms) { + if (preserveUnknownCustoms) { + assertNotNull(loadedMetaData.custom(IndexGraveyard.TYPE)); + assertThat(loadedMetaData.custom(IndexGraveyard.TYPE), instanceOf(MetaData.UnknownGatewayOnlyCustom.class)); + + // check that we reserialize unknown metadata correctly again + final Path tempdir = createTempDir(); + metaDataFormat(randomBoolean()).write(loadedMetaData, tempdir); + final MetaData reloadedMetaData = metaDataFormat(randomBoolean()).loadLatestState(logger, xContentRegistry(), tempdir); + assertThat(reloadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + } else { + assertNotNull(loadedMetaData.indexGraveyard()); + assertThat(loadedMetaData.indexGraveyard().getTombstones(), hasSize(0)); + } + } else { + assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + } // now corrupt all the latest ones and make sure we fail to load the state for (int i = 0; i < dirs.length; i++) { @@ -419,7 +454,7 @@ public class MetaDataStateFormatTests extends ESTestCase { writeAndReadStateSuccessfully(format, paths); } - private static MetaDataStateFormat metaDataFormat() { + private static MetaDataStateFormat metaDataFormat(boolean preserveUnknownCustoms) { return new MetaDataStateFormat(MetaData.GLOBAL_STATE_FILE_PREFIX) { @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { @@ -428,7 +463,7 @@ public class MetaDataStateFormatTests extends ESTestCase { @Override public MetaData fromXContent(XContentParser parser) throws IOException { - return MetaData.Builder.fromXContent(parser); + return MetaData.Builder.fromXContent(parser, preserveUnknownCustoms); } }; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java index d7799959f6c..084d965a6e7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java @@ -80,7 +80,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase { builder = metaDataBuilder.build().toXContent(builder, params); builder.endObject(); // deserialize metadata again - MetaData metaData = MetaData.Builder.fromXContent(createParser(builder)); + MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean()); // check that custom metadata still present assertThat(metaData.custom(licensesMetaData.getWriteableName()), notNullValue()); assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java index 0556b8535e4..75e5bc1073e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java @@ -64,7 +64,7 @@ public class WatcherMetaDataSerializationTests extends ESTestCase { builder = metaDataBuilder.build().toXContent(builder, params); builder.endObject(); // deserialize metadata again - MetaData metaData = MetaData.Builder.fromXContent(createParser(builder)); + MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean()); // check that custom metadata still present assertThat(metaData.custom(watcherMetaData.getWriteableName()), notNullValue()); assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());