honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-22 04:45:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add tool elasticsearch-node unsafe-bootstrap (#37696)

Browse Source 
elasticsearch-node tool helps to restore cluster if half or more of
master eligible nodes are lost. Of course, all bets are off, regarding
data consistency.

There are two parts of the tool: unsafe-bootstrap to be used when there
is still at least one master-eligible node alive and detach-cluster,
when there are no master-eligible nodes left.
This commit implements the first part.

Docs for the tool will be added separately as a part of #37812.
This commit is contained in:
Andrey Ershov 2019-01-24 19:25:55 +01:00 committed by GitHub
parent 289106a578
commit 4974684003
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 540 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
distribution/src/bin/elasticsearch-node Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
ES_MAIN_CLASS=org.elasticsearch.cluster.coordination.NodeToolCli \
"`dirname "$0"`"/elasticsearch-cli \
"$@"

BIN
distribution/src/bin/elasticsearch-node.bat Normal file
View File

Binary file not shown.

20
qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
View File

@ -301,7 +301,7 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
}
}
public void test100RepairIndexCliPackaging() {
public void test100ElasticsearchShardCliPackaging() {
assumeThat(installation, is(notNullValue()));
final Installation.Executables bin = installation.executables();
@ -318,4 +318,22 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
}
}
public void test110ElasticsearchNodeCliPackaging() {
assumeThat(installation, is(notNullValue()));
final Installation.Executables bin = installation.executables();
final Shell sh = new Shell();
Platforms.PlatformAction action = () -> {
final Result result = sh.run(bin.elasticsearchNode + " -h");
assertThat(result.stdout,
containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes"));
};
if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
Platforms.onLinux(action);
Platforms.onWindows(action);
}
}
}

3
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java
View File

@ -186,7 +186,8 @@ public class Archives {
"elasticsearch-env",
"elasticsearch-keystore",
"elasticsearch-plugin",
"elasticsearch-shard"
"elasticsearch-shard",
"elasticsearch-node"
).forEach(executable -> {
assertThat(es.bin(executable), file(File, owner, owner, p755));

1
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
View File

@ -102,6 +102,7 @@ public class Installation {
public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
public final Path elasticsearchShard = platformExecutable("elasticsearch-shard");
public final Path elasticsearchNode = platformExecutable("elasticsearch-node");
private Path platformExecutable(String name) {
final String platformExecutableName = Platforms.WINDOWS

3
qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
View File

@ -196,7 +196,8 @@ public class Packages {
"elasticsearch",
"elasticsearch-plugin",
"elasticsearch-keystore",
"elasticsearch-shard"
"elasticsearch-shard",
"elasticsearch-node"
).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
Stream.of(

1
qa/vagrant/src/test/resources/packaging/utils/packages.bash
View File

@ -104,6 +104,7 @@ verify_package_installation() {
assert_file "$ESHOME/bin/elasticsearch" f root root 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f root root 755
assert_file "$ESHOME/bin/elasticsearch-shard" f root root 755
assert_file "$ESHOME/bin/elasticsearch-node" f root root 755
assert_file "$ESHOME/lib" d root root 755
assert_file "$ESCONFIG" d root elasticsearch 2750
assert_file "$ESCONFIG/elasticsearch.keystore" f root elasticsearch 660

1
qa/vagrant/src/test/resources/packaging/utils/tar.bash
View File

@ -95,6 +95,7 @@ verify_archive_installation() {
assert_file "$ESHOME/bin/elasticsearch-keystore" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-plugin" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-shard" f elasticsearch elasticsearch 755
assert_file "$ESHOME/bin/elasticsearch-node" f elasticsearch elasticsearch 755
assert_file "$ESCONFIG" d elasticsearch elasticsearch 755
assert_file "$ESCONFIG/elasticsearch.yml" f elasticsearch elasticsearch 660
assert_file "$ESCONFIG/jvm.options" f elasticsearch elasticsearch 660

4
server/src/main/java/org/elasticsearch/cli/CommandLoggingConfigurator.java
View File

@ -27,13 +27,13 @@ import org.elasticsearch.common.settings.Settings;
* Holder class for method to configure logging without Elasticsearch configuration files for use in CLI tools that will not read such
* files.
*/
final class CommandLoggingConfigurator {
public final class CommandLoggingConfigurator {
/**
* Configures logging without Elasticsearch configuration files based on the system property "es.logger.level" only. As such, any
* logging will be written to the console.
*/
static void configureLoggingWithoutConfig() {
public static void configureLoggingWithoutConfig() {
// initialize default for es.logger.level because we will not read the log4j2.properties
final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name());
final Settings settings = Settings.builder().put("logger.level", loggerLevel).build();

44
server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.coordination;
import org.elasticsearch.cli.CommandLoggingConfigurator;
import org.elasticsearch.cli.MultiCommand;
import org.elasticsearch.cli.Terminal;
// NodeToolCli does not extend LoggingAwareCommand, because LoggingAwareCommand performs logging initialization
// after LoggingAwareCommand instance is constructed.
// It's too late for us, because before UnsafeBootstrapMasterCommand is added to the list of subcommands
// log4j2 initialization will happen, because it has static reference to Logger class.
// Even if we avoid making a static reference to Logger class, there is no nice way to avoid declaring
// UNSAFE_BOOTSTRAP, which depends on ClusterService, which in turn has static Logger.
// TODO execute CommandLoggingConfigurator.configureLoggingWithoutConfig() in the constructor of commands, not in beforeMain
public class NodeToolCli extends MultiCommand {
public NodeToolCli() {
super("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes", ()->{});
CommandLoggingConfigurator.configureLoggingWithoutConfig();
subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand());
}
public static void main(String[] args) throws Exception {
exit(new NodeToolCli().main(args, Terminal.DEFAULT));
}
}

195
server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java Normal file
View File

@ -0,0 +1,195 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.coordination;
import joptsimple.OptionSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.store.LockObtainFailedException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.Manifest;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.NodeMetaData;
import org.elasticsearch.node.Node;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.Locale;
import java.util.Objects;
public class UnsafeBootstrapMasterCommand extends EnvironmentAwareCommand {
private static final Logger logger = LogManager.getLogger(UnsafeBootstrapMasterCommand.class);
private final NamedXContentRegistry namedXContentRegistry;
static final String STOP_WARNING_MSG =
"--------------------------------------------------------------------------\n" +
"\n" +
" WARNING: Elasticsearch MUST be stopped before running this tool." +
"\n";
static final String CLUSTER_STATE_TERM_VERSION_MSG_FORMAT =
"Current node cluster state (term, version) pair is (%s, %s)";
static final String CONFIRMATION_MSG =
"--------------------------------------------------------------------------\n" +
"\n" +
"You should run this tool only if you have permanently lost half\n" +
"or more of the master-eligible nodes, and you cannot restore the cluster\n" +
"from a snapshot. This tool can result in arbitrary data loss and\n" +
"should be the last resort.\n" +
"If you have multiple survived master eligible nodes, consider running\n" +
"this tool on the node with the highest cluster state (term, version) pair.\n" +
"Do you want to proceed?\n";
static final String ABORTED_BY_USER_MSG = "aborted by user";
static final String NOT_MASTER_NODE_MSG = "unsafe-bootstrap tool can only be run on master eligible node";
static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?";
static final String NO_NODE_METADATA_FOUND_MSG = "no node meta data is found, node has not been started yet?";
static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?";
static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " +
"bootstrapped?";
static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?";
static final String EMPTY_LAST_COMMITTED_VOTING_CONFIG_MSG =
"last committed voting voting configuration is empty, cluster has never been bootstrapped?";
static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk";
static final String MASTER_NODE_BOOTSTRAPPED_MSG = "Master node was successfully bootstrapped";
static final Setting<String> UNSAFE_BOOTSTRAP =
ClusterService.USER_DEFINED_META_DATA.getConcreteSetting("cluster.metadata.unsafe-bootstrap");
UnsafeBootstrapMasterCommand() {
super("Forces the successful election of the current node after the permanent loss of the half or more master-eligible nodes");
namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
}
@Override
protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
terminal.println(STOP_WARNING_MSG);
Settings settings = env.settings();
terminal.println(Terminal.Verbosity.VERBOSE, "Checking node.master setting");
Boolean master = Node.NODE_MASTER_SETTING.get(settings);
if (master == false) {
throw new ElasticsearchException(NOT_MASTER_NODE_MSG);
}
final int nodeOrdinal = 0;
terminal.println(Terminal.Verbosity.VERBOSE, "Obtaining lock for node");
try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(nodeOrdinal, logger, env, Files::exists)) {
processNodePaths(logger, terminal, lock.getNodePaths());
} catch (LockObtainFailedException ex) {
throw new ElasticsearchException(
FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]");
}
terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
}
private void processNodePaths(Logger logger, Terminal terminal, NodeEnvironment.NodePath[] nodePaths) throws IOException {
final Path[] dataPaths =
Arrays.stream(nodePaths).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new);
if (dataPaths.length == 0) {
throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
}
terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
if (nodeMetaData == null) {
throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
}
String nodeId = nodeMetaData.nodeId();
terminal.println(Terminal.Verbosity.VERBOSE, "Current nodeId is " + nodeId);
terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
if (manifest == null) {
throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
}
if (manifest.isGlobalGenerationMissing()) {
throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
}
terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
dataPaths);
if (metaData == null) {
throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
}
final CoordinationMetaData coordinationMetaData = metaData.coordinationMetaData();
if (coordinationMetaData == null ||
coordinationMetaData.getLastCommittedConfiguration() == null ||
coordinationMetaData.getLastCommittedConfiguration().isEmpty()) {
throw new ElasticsearchException(EMPTY_LAST_COMMITTED_VOTING_CONFIG_MSG);
}
terminal.println(String.format(Locale.ROOT, CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, coordinationMetaData.term(),
metaData.version()));
terminal.println(CONFIRMATION_MSG);
String text = terminal.readText("Confirm [y/N] ");
if (text.equalsIgnoreCase("y") == false) {
throw new ElasticsearchException(ABORTED_BY_USER_MSG);
}
CoordinationMetaData newCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData)
.clearVotingConfigExclusions()
.lastAcceptedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId)))
.lastCommittedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId)))
.build();
terminal.println(Terminal.Verbosity.VERBOSE, "New coordination metadata is constructed " + newCoordinationMetaData);
Settings persistentSettings = Settings.builder()
.put(metaData.persistentSettings())
.put(UNSAFE_BOOTSTRAP.getKey(), true)
.build();
MetaData newMetaData = MetaData.builder(metaData)
.persistentSettings(persistentSettings)
.coordinationMetaData(newCoordinationMetaData)
.build();
writeNewMetaData(terminal, manifest, newMetaData, dataPaths);
}
private void writeNewMetaData(Terminal terminal, Manifest manifest, MetaData newMetaData, Path[] dataPaths) {
try {
terminal.println(Terminal.Verbosity.VERBOSE, "Writing new global metadata to disk");
long newGeneration = MetaData.FORMAT.write(newMetaData, dataPaths);
long newCurrentTerm = manifest.getCurrentTerm() + 1;
terminal.println(Terminal.Verbosity.VERBOSE, "Incrementing currentTerm. New value is " + newCurrentTerm);
Manifest newManifest = new Manifest(newCurrentTerm, manifest.getClusterStateVersion(), newGeneration,
manifest.getIndexGenerations());
terminal.println(Terminal.Verbosity.VERBOSE, "Writing new manifest file to disk");
Manifest.FORMAT.writeAndCleanup(newManifest, dataPaths);
terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up old metadata");
MetaData.FORMAT.cleanupOldFiles(newGeneration, dataPaths);
} catch (Exception e) {
terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up new metadata");
MetaData.FORMAT.cleanupOldFiles(manifest.getGlobalGeneration(), dataPaths);
throw new ElasticsearchException(WRITE_METADATA_EXCEPTION_MSG, e);
}
}
}

262
server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java Normal file
View File

@ -0,0 +1,262 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.coordination;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.Manifest;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.ElectMasterService;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.NodeMetaData;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.hamcrest.Matchers.containsString;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.discovery.zen:TRACE")
public class UnsafeBootstrapMasterIT extends ESIntegTestCase {
private int bootstrapNodeId;
@Before
public void resetBootstrapNodeId() {
bootstrapNodeId = -1;
}
/**
* Performs cluster bootstrap when node with id bootstrapNodeId is started.
* Any node of the batch could be selected as bootstrap target.
*/
@Override
protected List<Settings> addExtraClusterBootstrapSettings(List<Settings> allNodesSettings) {
if (internalCluster().size() + allNodesSettings.size() == bootstrapNodeId) {
List<String> nodeNames = new ArrayList<>();
Collections.addAll(nodeNames, internalCluster().getNodeNames());
allNodesSettings.forEach(settings -> nodeNames.add(Node.NODE_NAME_SETTING.get(settings)));
List<Settings> newSettings = new ArrayList<>();
int bootstrapIndex = randomInt(allNodesSettings.size() - 1);
for (int i = 0; i < allNodesSettings.size(); i++) {
Settings nodeSettings = allNodesSettings.get(i);
if (i == bootstrapIndex) {
newSettings.add(Settings.builder().put(nodeSettings)
.putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames)
.build());
} else {
newSettings.add(nodeSettings);
}
}
return newSettings;
}
return allNodesSettings;
}
private MockTerminal executeCommand(Environment environment, boolean abort) throws Exception {
final UnsafeBootstrapMasterCommand command = new UnsafeBootstrapMasterCommand();
final MockTerminal terminal = new MockTerminal();
final OptionParser parser = new OptionParser();
final OptionSet options = parser.parse();
final String input;
if (abort) {
input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1));
} else {
input = randomBoolean() ? "y" : "Y";
}
terminal.addTextInput(input);
try {
command.execute(terminal, options, environment);
assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.MASTER_NODE_BOOTSTRAPPED_MSG));
} finally {
assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.STOP_WARNING_MSG));
}
return terminal;
}
private MockTerminal executeCommand(Environment environment) throws Exception {
return executeCommand(environment, false);
}
private void expectThrows(ThrowingRunnable runnable, String message) {
ElasticsearchException ex = expectThrows(ElasticsearchException.class, runnable);
assertThat(ex.getMessage(), containsString(message));
}
public void testNotMasterEligible() {
final Environment environment = TestEnvironment.newEnvironment(Settings.builder()
.put(internalCluster().getDefaultSettings())
.put(Node.NODE_MASTER_SETTING.getKey(), false)
.build());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NOT_MASTER_NODE_MSG);
}
public void testNoDataFolder() {
final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_NODE_FOLDER_FOUND_MSG);
}
public void testNodeLocked() throws IOException {
Settings envSettings = buildEnvSettings(Settings.EMPTY);
Environment environment = TestEnvironment.newEnvironment(envSettings);
try (NodeEnvironment ignored = new NodeEnvironment(envSettings, environment)) {
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG);
}
}
public void testNoNodeMetaData() throws IOException {
Settings envSettings = buildEnvSettings(Settings.EMPTY);
Environment environment = TestEnvironment.newEnvironment(envSettings);
try (NodeEnvironment nodeEnvironment = new NodeEnvironment(envSettings, environment)) {
NodeMetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths());
}
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_NODE_METADATA_FOUND_MSG);
}
public void testNotBootstrappedCluster() throws Exception {
internalCluster().startNode(
Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup
.build());
assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().setLocal(true)
.execute().actionGet().getState();
assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID));
});
internalCluster().stopRandomDataNode();
Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.GLOBAL_GENERATION_MISSING_MSG);
}
public void testNoManifestFile() throws IOException {
bootstrapNodeId = 1;
internalCluster().startNode(Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
ensureStableCluster(1);
NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
internalCluster().stopRandomDataNode();
Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
Manifest.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_MANIFEST_FILE_FOUND_MSG);
}
public void testNoMetaData() throws IOException {
bootstrapNodeId = 1;
internalCluster().startNode(Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
ensureStableCluster(1);
NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
internalCluster().stopRandomDataNode();
Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
MetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_GLOBAL_METADATA_MSG);
}
public void testAbortedByUser() throws IOException {
bootstrapNodeId = 1;
internalCluster().startNode(Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
ensureStableCluster(1);
internalCluster().stopRandomDataNode();
Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
expectThrows(() -> executeCommand(environment, true), UnsafeBootstrapMasterCommand.ABORTED_BY_USER_MSG);
}
public void test3MasterNodes2Failed() throws Exception {
bootstrapNodeId = 3;
List<String> masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
String dataNode = internalCluster().startDataOnlyNode(Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
createIndex("test");
Client dataNodeClient = internalCluster().client(dataNode);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1)));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(2)));
assertBusy(() -> {
ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true)
.execute().actionGet().getState();
assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID));
});
final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings());
expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG);
NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0)));
MockTerminal terminal = executeCommand(environment);
MetaData metaData = MetaData.FORMAT.loadLatestState(logger, xContentRegistry(), nodeEnvironment.nodeDataPaths());
assertThat(terminal.getOutput(), containsString(
String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT,
metaData.coordinationMetaData().term(), metaData.version())));
internalCluster().startMasterOnlyNode(Settings.builder()
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)
.build());
assertBusy(() -> {
ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true)
.execute().actionGet().getState();
assertFalse(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID));
assertTrue(state.metaData().persistentSettings().getAsBoolean(UnsafeBootstrapMasterCommand.UNSAFE_BOOTSTRAP.getKey(), false));
});
ensureGreen("test");
}
}

8
test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
View File

@ -928,11 +928,15 @@ public abstract class ESTestCase extends LuceneTestCase {
return newNodeEnvironment(Settings.EMPTY);
}
public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException {
Settings build = Settings.builder()
public Settings buildEnvSettings(Settings settings) {
return Settings.builder()
.put(settings)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
.putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).build();
}
public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException {
Settings build = buildEnvSettings(settings);
return new NodeEnvironment(build, TestEnvironment.newEnvironment(build));
}