From 4b0581f182467b6f40945e7f385a3a0a7ba80acc Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 14 Jan 2020 18:33:53 +0100 Subject: [PATCH] Remove custom metadata tool (#50813) Adds a command-line tool to remove broken custom metadata from the cluster state. Relates to #48701 --- docs/reference/commands/node-tool.asciidoc | 55 +++++++- .../cluster/coordination/NodeToolCli.java | 1 + .../coordination/RemoveCustomsCommand.java | 102 ++++++++++++++ .../coordination/RemoveCustomsCommandIT.java | 125 ++++++++++++++++++ 4 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc index d6342ee7810..4a1367832c8 100644 --- a/docs/reference/commands/node-tool.asciidoc +++ b/docs/reference/commands/node-tool.asciidoc @@ -20,7 +20,7 @@ bin/elasticsearch-node repurpose|unsafe-bootstrap|detach-cluster|override-versio [float] === Description -This tool has five modes: +This tool has a number of modes: * `elasticsearch-node repurpose` can be used to delete unwanted data from a node if it used to be a <> or a @@ -28,8 +28,12 @@ This tool has five modes: or other of these roles. * `elasticsearch-node remove-settings` can be used to remove persistent settings - from the cluster state in case where it contains incompatible settings that - prevent the cluster from forming. + from the cluster state in case where it contains incompatible settings that + prevent the cluster from forming. + +* `elasticsearch-node remove-customs` can be used to remove custom metadata + from the cluster state in case where it contains broken metadata that + prevents the cluster state from being loaded. * `elasticsearch-node unsafe-bootstrap` can be used to perform _unsafe cluster bootstrapping_. It forces one of the nodes to form a brand-new cluster on @@ -100,6 +104,24 @@ The intended use is: * Repeat for all other master-eligible nodes * Start the nodes +[float] +==== Removing custom metadata from the cluster state + +There may be situations where a node contains custom metadata, typically +provided by plugins, that prevent the node from starting up and loading +the cluster from disk. + +The `elasticsearch-node remove-customs` tool allows you to forcefully remove +the problematic custom metadata. The tool takes a list of custom metadata names +as parameters that should be removed, and also supports wildcard patterns. + +The intended use is: + +* Stop the node +* Run `elasticsearch-node remove-customs name-of-custom-to-remove` on the node +* Repeat for all other master-eligible nodes +* Start the nodes + [float] ==== Recovering data after a disaster @@ -411,6 +433,33 @@ You can also use wildcards to remove multiple settings, for example using node$ ./bin/elasticsearch-node remove-settings xpack.monitoring.* ---- +[float] +==== Removing custom metadata from the cluster state + +If the on-disk cluster state contains custom metadata that prevents the node +from starting up and loading the cluster state, you can run the following +commands to remove this custom metadata. + +[source,txt] +---- +node$ ./bin/elasticsearch-node remove-customs snapshot_lifecycle + + WARNING: Elasticsearch MUST be stopped before running this tool. + +The following customs will be removed: +snapshot_lifecycle + +You should only run this tool if you have broken custom metadata in the +cluster state that prevents the cluster state from being loaded. +This tool can cause data loss and its use should be your last resort. + +Do you want to proceed? + +Confirm [y/N] y + +Customs were successfully removed from the cluster state +---- + [float] ==== Unsafe cluster bootstrapping diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java index 29c03dbb8c4..7c96f631be6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java @@ -42,6 +42,7 @@ public class NodeToolCli extends MultiCommand { subcommands.put("detach-cluster", new DetachClusterCommand()); subcommands.put("override-version", new OverrideNodeVersionCommand()); subcommands.put("remove-settings", new RemoveSettingsCommand()); + subcommands.put("remove-customs", new RemoveCustomsCommand()); } public static void main(String[] args) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java new file mode 100644 index 00000000000..35fcd8b517d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.env.Environment; +import org.elasticsearch.gateway.PersistedClusterStateService; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +public class RemoveCustomsCommand extends ElasticsearchNodeCommand { + + static final String CUSTOMS_REMOVED_MSG = "Customs were successfully removed from the cluster state"; + static final String CONFIRMATION_MSG = + DELIMITER + + "\n" + + "You should only run this tool if you have broken custom metadata in the\n" + + "cluster state that prevents the cluster state from being loaded.\n" + + "This tool can cause data loss and its use should be your last resort.\n" + + "\n" + + "Do you want to proceed?\n"; + + private final OptionSpec arguments; + + public RemoveCustomsCommand() { + super("Removes custom metadata from the cluster state"); + arguments = parser.nonOptions("custom metadata names"); + } + + @Override + protected void processNodePaths(Terminal terminal, Path[] dataPaths, int nodeLockId, OptionSet options, Environment env) + throws IOException, UserException { + final List customsToRemove = arguments.values(options); + if (customsToRemove.isEmpty()) { + throw new UserException(ExitCodes.USAGE, "Must supply at least one custom metadata name to remove"); + } + + final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths); + + terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state"); + final Tuple termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env); + final ClusterState oldClusterState = termAndClusterState.v2(); + terminal.println(Terminal.Verbosity.VERBOSE, "custom metadata names: " + oldClusterState.metaData().customs().keys()); + final MetaData.Builder metaDataBuilder = MetaData.builder(oldClusterState.metaData()); + for (String customToRemove : customsToRemove) { + boolean matched = false; + for (ObjectCursor customKeyCur : oldClusterState.metaData().customs().keys()) { + final String customKey = customKeyCur.value; + if (Regex.simpleMatch(customToRemove, customKey)) { + metaDataBuilder.removeCustom(customKey); + if (matched == false) { + terminal.println("The following customs will be removed:"); + } + matched = true; + terminal.println(customKey); + } + } + if (matched == false) { + throw new UserException(ExitCodes.USAGE, + "No custom metadata matching [" + customToRemove + "] were found on this node"); + } + } + final ClusterState newClusterState = ClusterState.builder(oldClusterState).metaData(metaDataBuilder.build()).build(); + terminal.println(Terminal.Verbosity.VERBOSE, + "[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"); + + confirm(terminal, CONFIRMATION_MSG); + + try (PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter()) { + writer.writeFullStateAndCommit(termAndClusterState.v1(), newClusterState); + } + + terminal.println(CUSTOMS_REMOVED_MSG); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java new file mode 100644 index 00000000000..867f05131f6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import joptsimple.OptionSet; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.hamcrest.Matchers.containsString; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +public class RemoveCustomsCommandIT extends ESIntegTestCase { + + public void testRemoveCustomsAbortedByUser() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + String node = internalCluster().startNode(); + Settings dataPathSettings = internalCluster().dataPathSettings(node); + ensureStableCluster(1); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment( + Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()); + expectThrows(() -> removeCustoms(environment, true, new String[]{ "index-graveyard" }), + ElasticsearchNodeCommand.ABORTED_BY_USER_MSG); + } + + public void testRemoveCustomsSuccessful() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + String node = internalCluster().startNode(); + createIndex("test"); + client().admin().indices().prepareDelete("test").get(); + assertEquals(1, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size()); + Settings dataPathSettings = internalCluster().dataPathSettings(node); + ensureStableCluster(1); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment( + Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()); + MockTerminal terminal = removeCustoms(environment, false, + randomBoolean() ? + new String[]{ "index-graveyard" } : + new String[]{ "index-*" } + ); + assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CUSTOMS_REMOVED_MSG)); + assertThat(terminal.getOutput(), containsString("The following customs will be removed:")); + assertThat(terminal.getOutput(), containsString("index-graveyard")); + + internalCluster().startNode(dataPathSettings); + assertEquals(0, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size()); + } + + public void testCustomDoesNotMatch() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + String node = internalCluster().startNode(); + createIndex("test"); + client().admin().indices().prepareDelete("test").get(); + assertEquals(1, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size()); + Settings dataPathSettings = internalCluster().dataPathSettings(node); + ensureStableCluster(1); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment( + Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()); + UserException ex = expectThrows(UserException.class, () -> removeCustoms(environment, false, + new String[]{ "index-greveyard-with-typos" })); + assertThat(ex.getMessage(), containsString("No custom metadata matching [index-greveyard-with-typos] were " + + "found on this node")); + } + + private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort, String... args) + throws Exception { + final MockTerminal terminal = new MockTerminal(); + final OptionSet options = command.getParser().parse(args); + final String input; + + if (abort) { + input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1)); + } else { + input = randomBoolean() ? "y" : "Y"; + } + + terminal.addTextInput(input); + + try { + command.execute(terminal, options, environment); + } finally { + assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG)); + } + + return terminal; + } + + private MockTerminal removeCustoms(Environment environment, boolean abort, String... args) throws Exception { + final MockTerminal terminal = executeCommand(new RemoveCustomsCommand(), environment, abort, args); + assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CONFIRMATION_MSG)); + assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CUSTOMS_REMOVED_MSG)); + return terminal; + } + + private void expectThrows(ThrowingRunnable runnable, String message) { + ElasticsearchException ex = expectThrows(ElasticsearchException.class, runnable); + assertThat(ex.getMessage(), containsString(message)); + } +}