Make elasticsearch-node tools custom metadata-aware (#48390)

The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently
unaware of plugins and will therefore drop custom metadata from the cluster state once the
state is written out again (as it skips over the custom metadata that it can't read). This commit
preserves unknown customs when editing on-disk metadata through the elasticsearch-node
command-line tools.

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -418,7 +418,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         if (plugins.isEmpty() == false) {
             logToProcessStdout("Installing " + plugins.size() + " plugins");
-            plugins.forEach(plugin -> runElaticsearchBinScript(
+            plugins.forEach(plugin -> runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", plugin.toString())
             );
@@ -426,7 +426,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
             LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
-            runElaticsearchBinScript(
+            runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", "x-pack"
             );
@@ -434,10 +434,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
-            runElaticsearchBinScript("elasticsearch-keystore", "create");
+            runElasticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
-                runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
+                runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
             );
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
@@ -446,7 +446,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
                 if (file.exists() == false) {
                     throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
                 }
-                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+                runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
             }
         }
 
@@ -463,7 +463,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         if (credentials.isEmpty() == false) {
             logToProcessStdout("Setting up " + credentials.size() + " users");
 
-            credentials.forEach(paramMap -> runElaticsearchBinScript(
+            credentials.forEach(paramMap -> runElasticsearchBinScript(
                 getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -592,7 +592,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         credentials.add(cred);
     }
 
-    private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
+    private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
         if (
             Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
                 Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -632,8 +632,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         }
     }
 
-    private void runElaticsearchBinScript(String tool, String... args) {
+    private void runElasticsearchBinScript(String tool, String... args) {
-        runElaticsearchBinScriptWithInput("", tool, args);
+        runElasticsearchBinScriptWithInput("", tool, args);
     }
 
     private Map<String, String> getESEnvironment() {

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -819,7 +819,7 @@ public final class XContentBuilder implements Closeable, Flushable {
         } else if (value instanceof Map) {
             @SuppressWarnings("unchecked")
             final Map<String, ?> valueMap = (Map<String, ?>) value;
-            map(valueMap, ensureNoSelfReferences);
+            map(valueMap, ensureNoSelfReferences, true);
         } else if (value instanceof Iterable) {
             value((Iterable<?>) value, ensureNoSelfReferences);
         } else if (value instanceof Object[]) {
@@ -867,10 +867,15 @@ public final class XContentBuilder implements Closeable, Flushable {
     }
 
     public XContentBuilder map(Map<String, ?> values) throws IOException {
-        return map(values, true);
+        return map(values, true, true);
     }
 
-    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences) throws IOException {
+    /** writes a map without the start object and end object headers */
+    public XContentBuilder mapContents(Map<String, ?> values) throws IOException {
+        return map(values, true, false);
+    }
+
+    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException {
         if (values == null) {
             return nullValue();
         }
@@ -881,13 +886,17 @@ public final class XContentBuilder implements Closeable, Flushable {
             ensureNoSelfReferences(values);
         }
 
-        startObject();
+        if (writeStartAndEndHeaders) {
+            startObject();
+        }
         for (Map.Entry<String, ?> value : values.entrySet()) {
             field(value.getKey());
             // pass ensureNoSelfReferences=false as we already performed the check at a higher level
             unknownValue(value.getValue(), false);
         }
-        endObject();
+        if (writeStartAndEndHeaders) {
+            endObject();
+        }
         return this;
     }
 

server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java

@@ -88,14 +88,19 @@ public abstract class EnvironmentAwareCommand extends Command {
 
     /** Create an {@link Environment} for the command to use. Overrideable for tests. */
     protected Environment createEnv(final Map<String, String> settings) throws UserException {
+        return createEnv(Settings.EMPTY, settings);
+    }
+
+    /** Create an {@link Environment} for the command to use. Overrideable for tests. */
+    protected final Environment createEnv(final Settings baseSettings, final Map<String, String> settings) throws UserException {
         final String esPathConf = System.getProperty("es.path.conf");
         if (esPathConf == null) {
             throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set");
         }
-        return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings,
+        return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings,
-                getConfigPath(esPathConf),
+            getConfigPath(esPathConf),
-                // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
+            // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
-                () -> System.getenv("HOSTNAME"));
+            () -> System.getenv("HOSTNAME"));
     }
 
     @SuppressForbidden(reason = "need path to construct environment")

server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java

@@ -27,7 +27,6 @@ import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.Tuple;
@@ -43,7 +42,6 @@ import java.util.Objects;
 
 public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
     private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
-    protected final NamedXContentRegistry namedXContentRegistry;
     protected static final String DELIMITER = "------------------------------------------------------------------------\n";
 
     static final String STOP_WARNING_MSG =
@@ -65,7 +63,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
         super(description);
         nodeOrdinalOption = parser.accepts("ordinal", "Optional node ordinal, 0 if not specified")
                 .withRequiredArg().ofType(Integer.class);
-        namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
     }
 
     protected void processNodePathsWithLock(Terminal terminal, OptionSet options, Environment env) throws IOException {
@@ -88,7 +85,7 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
 
     protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
 
         if (manifest == null) {
             throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
@@ -97,8 +94,8 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
             throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
         }
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
-        final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
+        final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration(
-                dataPaths);
+            logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths);
         if (metaData == null) {
             throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
         }

server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeMetaData;
 import org.elasticsearch.node.Node;
@@ -84,7 +85,7 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand {
 
     protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
-        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
         if (nodeMetaData == null) {
             throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
         }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -45,7 +45,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1513,8 +1512,6 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
 
         @Override
         public IndexMetaData fromXContent(XContentParser parser) throws IOException {
-            assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
-                    : "loading index metadata requires a working named xcontent registry";
             return Builder.fromXContent(parser);
         }
     };

server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java

@@ -813,7 +813,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
     }
 
     public static MetaData fromXContent(XContentParser parser) throws IOException {
-        return Builder.fromXContent(parser);
+        return Builder.fromXContent(parser, false);
     }
 
     @Override
@@ -1353,7 +1353,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
             builder.endObject();
         }
 
-        public static MetaData fromXContent(XContentParser parser) throws IOException {
+        public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException {
             Builder builder = new Builder();
 
             // we might get here after the meta-data element, or on a fresh parser
@@ -1403,8 +1403,13 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
                             Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
-                            logger.warn("Skipping unknown custom object with type {}", currentFieldName);
+                            if (preserveUnknownCustoms) {
-                            parser.skipChildren();
+                                logger.warn("Adding unknown custom object with type {}", currentFieldName);
+                                builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered()));
+                            } else {
+                                logger.warn("Skipping unknown custom object with type {}", currentFieldName);
+                                parser.skipChildren();
+                            }
                         }
                     }
                 } else if (token.isValue()) {
@@ -1425,6 +1430,45 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
         }
     }
 
+    public static class UnknownGatewayOnlyCustom implements Custom {
+
+        private final Map<String, Object> contents;
+
+        UnknownGatewayOnlyCustom(Map<String, Object> contents) {
+            this.contents = contents;
+        }
+
+        @Override
+        public EnumSet<XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
+        }
+
+        @Override
+        public Diff<Custom> diff(Custom previousState) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.mapContents(contents);
+        }
+    }
+
     private static final ToXContent.Params FORMAT_PARAMS;
     static {
         Map<String, String> params = new HashMap<>(2);
@@ -1436,16 +1480,25 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
     /**
      * State format for {@link MetaData} to write to and load from disk
      */
-    public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
+    public static final MetaDataStateFormat<MetaData> FORMAT = createMetaDataStateFormat(false);
 
-        @Override
+    /**
-        public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+     * Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs
-            Builder.toXContent(state, builder, FORMAT_PARAMS);
+     */
-        }
+    public static final MetaDataStateFormat<MetaData> FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true);
 
-        @Override
+    private static MetaDataStateFormat<MetaData> createMetaDataStateFormat(boolean preserveUnknownCustoms) {
-        public MetaData fromXContent(XContentParser parser) throws IOException {
+        return new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
-            return Builder.fromXContent(parser);
+
-        }
+            @Override
-    };
+            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+                Builder.toXContent(state, builder, FORMAT_PARAMS);
+            }
+
+            @Override
+            public MetaData fromXContent(XContentParser parser) throws IOException {
+                return Builder.fromXContent(parser, preserveUnknownCustoms);
+            }
+        };
+    }
 }

server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java

@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.WriteStateException;
 
@@ -165,7 +166,7 @@ public class NodeRepurposeCommand extends ElasticsearchNodeCommand {
             indexPaths[i] = nodePaths[i].resolve(uuid);
         }
         try {
-            IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths);
+            IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths);
             return metaData.getIndex().getName();
         } catch (Exception e) {
             return "no name for uuid: " + uuid + ": " + e;
@@ -194,7 +195,7 @@ public class NodeRepurposeCommand extends ElasticsearchNodeCommand {
 
     private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest");
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
 
         if (manifest == null) {
             terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE);

server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java

@@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -74,7 +75,7 @@ public class OverrideNodeVersionCommand extends ElasticsearchNodeCommand {
     protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new);
         final NodeMetaData nodeMetaData
-            = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths);
+            = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths);
         if (nodeMetaData == null) {
             throw new ElasticsearchException(NO_METADATA_MESSAGE);
         }

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java

@@ -365,7 +365,7 @@ public class MetaDataTests extends ESTestCase {
                 .endObject()
             .endObject());
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) {
-            MetaData.Builder.fromXContent(parser);
+            MetaData.Builder.fromXContent(parser, randomBoolean());
             fail();
         } catch (IllegalArgumentException e) {
             assertEquals("Unexpected field [random]", e.getMessage());

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

@@ -147,7 +147,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
 
         String metaDataSource = MetaData.Builder.toXContent(metaData);
 
-        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource));
+        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false);
 
         IndexMetaData indexMetaData = parsedMetaData.index("test1");
         assertThat(indexMetaData.primaryTerm(0), equalTo(1L));

server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java

@@ -61,6 +61,8 @@ import java.util.Set;
 import java.util.stream.StreamSupport;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -80,7 +82,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
 
             @Override
             public MetaData fromXContent(XContentParser parser) throws IOException {
-                return MetaData.Builder.fromXContent(parser);
+                return MetaData.Builder.fromXContent(parser, false);
             }
         };
         Path tmp = createTempDir();
@@ -233,7 +235,23 @@ public class MetaDataStateFormatTests extends ESTestCase {
         }
     }
 
-    public void testLoadState() throws IOException {
+    public void testLoadStateWithoutMissingCustoms() throws IOException {
+        runLoadStateTest(false, false);
+    }
+
+    public void testLoadStateWithoutMissingCustomsButPreserved() throws IOException {
+        runLoadStateTest(false, true);
+    }
+
+    public void testLoadStateWithMissingCustomsButPreserved() throws IOException {
+        runLoadStateTest(true, true);
+    }
+
+    public void testLoadStateWithMissingCustomsAndNotPreserved() throws IOException {
+        runLoadStateTest(true, false);
+    }
+
+    private void runLoadStateTest(boolean hasMissingCustoms, boolean preserveUnknownCustoms) throws IOException {
         final Path[] dirs = new Path[randomIntBetween(1, 5)];
         int numStates = randomIntBetween(1, 5);
         List<MetaData> meta = new ArrayList<>();
@@ -241,7 +259,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             meta.add(randomMeta());
         }
         Set<Path> corruptedFiles = new HashSet<>();
-        MetaDataStateFormat<MetaData> format = metaDataFormat();
+        MetaDataStateFormat<MetaData> format = metaDataFormat(preserveUnknownCustoms);
         for (int i = 0; i < dirs.length; i++) {
             dirs[i] = createTempDir();
             Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
@@ -258,11 +276,12 @@ public class MetaDataStateFormatTests extends ESTestCase {
         }
         List<Path> dirList = Arrays.asList(dirs);
         Collections.shuffle(dirList, random());
-        MetaData loadedMetaData = format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
+        MetaData loadedMetaData = format.loadLatestState(logger, hasMissingCustoms ?
+            NamedXContentRegistry.EMPTY : xContentRegistry(), dirList.toArray(new Path[0]));
         MetaData latestMetaData = meta.get(numStates-1);
         assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_")));
         assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID()));
-        ImmutableOpenMap<String,IndexMetaData> indices = loadedMetaData.indices();
+        ImmutableOpenMap<String, IndexMetaData> indices = loadedMetaData.indices();
         assertThat(indices.size(), equalTo(latestMetaData.indices().size()));
         for (IndexMetaData original : latestMetaData) {
             IndexMetaData deserialized = indices.get(original.getIndex().getName());
@@ -275,7 +294,23 @@ public class MetaDataStateFormatTests extends ESTestCase {
         }
 
         // make sure the index tombstones are the same too
-        assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
+        if (hasMissingCustoms) {
+            if (preserveUnknownCustoms) {
+                assertNotNull(loadedMetaData.custom(IndexGraveyard.TYPE));
+                assertThat(loadedMetaData.custom(IndexGraveyard.TYPE), instanceOf(MetaData.UnknownGatewayOnlyCustom.class));
+
+                // check that we reserialize unknown metadata correctly again
+                final Path tempdir = createTempDir();
+                metaDataFormat(randomBoolean()).write(loadedMetaData, tempdir);
+                final MetaData reloadedMetaData = metaDataFormat(randomBoolean()).loadLatestState(logger, xContentRegistry(), tempdir);
+                assertThat(reloadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
+            } else {
+                assertNotNull(loadedMetaData.indexGraveyard());
+                assertThat(loadedMetaData.indexGraveyard().getTombstones(), hasSize(0));
+            }
+        }  else {
+            assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard()));
+        }
 
         // now corrupt all the latest ones and make sure we fail to load the state
         for (int i = 0; i < dirs.length; i++) {
@@ -419,7 +454,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         writeAndReadStateSuccessfully(format, paths);
     }
 
-    private static MetaDataStateFormat<MetaData> metaDataFormat() {
+    private static MetaDataStateFormat<MetaData> metaDataFormat(boolean preserveUnknownCustoms) {
         return new MetaDataStateFormat<MetaData>(MetaData.GLOBAL_STATE_FILE_PREFIX) {
             @Override
             public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
@@ -428,7 +463,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
 
             @Override
             public MetaData fromXContent(XContentParser parser) throws IOException {
-                return MetaData.Builder.fromXContent(parser);
+                return MetaData.Builder.fromXContent(parser, preserveUnknownCustoms);
             }
         };
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java

@@ -80,7 +80,7 @@ public class LicensesMetaDataSerializationTests extends ESTestCase {
         builder = metaDataBuilder.build().toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
-        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder));
+        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean());
         // check that custom metadata still present
         assertThat(metaData.custom(licensesMetaData.getWriteableName()), notNullValue());
         assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java

@@ -64,7 +64,7 @@ public class WatcherMetaDataSerializationTests extends ESTestCase {
         builder = metaDataBuilder.build().toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
-        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder));
+        MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean());
         // check that custom metadata still present
         assertThat(metaData.custom(watcherMetaData.getWriteableName()), notNullValue());
         assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());