Refactor bootstrap checks
This commit refactors the bootstrap checks into a dedicated class. The refactoring provides a model for different limits per operating system, and provides a model for unit tests for individual checks. Closes #16844
This commit is contained in:
Jason Tedor
parent
aa0ef84f5a
commit
9ad5919002
|
|
@ -22,7 +22,6 @@ package org.elasticsearch.bootstrap;
|
|||
import org.apache.lucene.util.Constants;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.StringHelper;
|
||||
import org.elasticsearch.Build;
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.PidFile;
|
||||
|
|
@ -33,8 +32,6 @@ import org.elasticsearch.common.inject.CreationException;
|
|||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.LogConfigurator;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.common.network.NetworkService;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.monitor.jvm.JvmInfo;
|
||||
|
|
@ -42,17 +39,12 @@ import org.elasticsearch.monitor.os.OsProbe;
|
|||
import org.elasticsearch.monitor.process.ProcessProbe;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.node.internal.InternalSettingsPreparer;
|
||||
import org.elasticsearch.transport.TransportSettings;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
|
||||
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
|
||||
|
|
@ -189,7 +181,8 @@ final class Bootstrap {
|
|||
.put(settings)
|
||||
.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
|
||||
.build();
|
||||
enforceOrLogLimits(nodeSettings);
|
||||
|
||||
BootstrapCheck.check(nodeSettings);
|
||||
|
||||
node = new Node(nodeSettings);
|
||||
}
|
||||
|
|
@ -349,50 +342,4 @@ final class Bootstrap {
|
|||
}
|
||||
}
|
||||
|
||||
static final Set<Setting> ENFORCE_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
|
||||
TransportSettings.BIND_HOST,
|
||||
TransportSettings.HOST,
|
||||
TransportSettings.PUBLISH_HOST,
|
||||
NetworkService.GLOBAL_NETWORK_HOST_SETTING,
|
||||
NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
|
||||
NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING
|
||||
)));
|
||||
|
||||
private static boolean enforceLimits(Settings settings) {
|
||||
if (Build.CURRENT.isSnapshot()) {
|
||||
return false;
|
||||
}
|
||||
for (Setting setting : ENFORCE_SETTINGS) {
|
||||
if (setting.exists(settings)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void enforceOrLogLimits(Settings settings) { // pkg private for testing
|
||||
/* We enforce limits once any network host is configured. In this case we assume the node is running in production
|
||||
* and all production limit checks must pass. This should be extended as we go to settings like:
|
||||
* - discovery.zen.minimum_master_nodes
|
||||
* - discovery.zen.ping.unicast.hosts is set if we use zen disco
|
||||
* - ensure we can write in all data directories
|
||||
* - fail if mlockall failed and was configured
|
||||
* - fail if vm.max_map_count is under a certain limit (not sure if this works cross platform)
|
||||
* - fail if the default cluster.name is used, if this is setup on network a real clustername should be used?*/
|
||||
final boolean enforceLimits = enforceLimits(settings);
|
||||
final ESLogger logger = Loggers.getLogger(Bootstrap.class);
|
||||
final long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount();
|
||||
if (maxFileDescriptorCount != -1) {
|
||||
final int fileDescriptorCountThreshold = (1 << 16);
|
||||
if (maxFileDescriptorCount < fileDescriptorCountThreshold) {
|
||||
if (enforceLimits){
|
||||
throw new IllegalStateException("max file descriptors [" + maxFileDescriptorCount
|
||||
+ "] for elasticsearch process likely too low, increase it to at least [" + fileDescriptorCountThreshold +"]");
|
||||
}
|
||||
logger.warn(
|
||||
"max file descriptors [{}] for elasticsearch process likely too low, consider increasing to at least [{}]",
|
||||
maxFileDescriptorCount, fileDescriptorCountThreshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.bootstrap;
|
||||
|
||||
import org.apache.lucene.util.Constants;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.common.network.NetworkService;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.monitor.process.ProcessProbe;
|
||||
import org.elasticsearch.transport.TransportSettings;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* We enforce limits once any network host is configured. In this case we assume the node is running in production
|
||||
* and all production limit checks must pass. This should be extended as we go to settings like:
|
||||
* - discovery.zen.minimum_master_nodes
|
||||
* - discovery.zen.ping.unicast.hosts is set if we use zen disco
|
||||
* - ensure we can write in all data directories
|
||||
* - fail if mlockall failed and was configured
|
||||
* - fail if vm.max_map_count is under a certain limit (not sure if this works cross platform)
|
||||
* - fail if the default cluster.name is used, if this is setup on network a real clustername should be used?
|
||||
*/
|
||||
final class BootstrapCheck {
|
||||
|
||||
private BootstrapCheck() {
|
||||
}
|
||||
|
||||
/**
|
||||
* checks the current limits against the snapshot or release build
|
||||
* checks
|
||||
*
|
||||
* @param settings the current node settings
|
||||
*/
|
||||
public static void check(final Settings settings) {
|
||||
check(enforceLimits(settings), checks());
|
||||
}
|
||||
|
||||
/**
|
||||
* executes the provided checks and fails the node if
|
||||
* enforceLimits is true, otherwise logs warnings
|
||||
*
|
||||
* @param enforceLimits true if the checks should be enforced or
|
||||
* warned
|
||||
* @param checks the checks to execute
|
||||
*/
|
||||
// visible for testing
|
||||
static void check(boolean enforceLimits, List<Check> checks) {
|
||||
final ESLogger logger = Loggers.getLogger(BootstrapCheck.class);
|
||||
|
||||
for (Check check : checks) {
|
||||
final boolean fail = check.check();
|
||||
if (fail) {
|
||||
if (enforceLimits) {
|
||||
throw new RuntimeException(check.errorMessage());
|
||||
} else {
|
||||
logger.warn(check.errorMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The set of settings such that if any are set for the node, then
|
||||
* the checks are enforced
|
||||
*
|
||||
* @return the enforcement settings
|
||||
*/
|
||||
// visible for testing
|
||||
static Set<Setting> enforceSettings() {
|
||||
return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
|
||||
TransportSettings.BIND_HOST,
|
||||
TransportSettings.HOST,
|
||||
TransportSettings.PUBLISH_HOST,
|
||||
NetworkService.GLOBAL_NETWORK_HOST_SETTING,
|
||||
NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
|
||||
NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING
|
||||
)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests if the checks should be enforced
|
||||
*
|
||||
* @param settings the current node settings
|
||||
* @return true if the checks should be enforced
|
||||
*/
|
||||
// visible for testing
|
||||
static boolean enforceLimits(final Settings settings) {
|
||||
return enforceSettings().stream().anyMatch(s -> s.exists(settings));
|
||||
}
|
||||
|
||||
// the list of checks to execute
|
||||
private static List<Check> checks() {
|
||||
FileDescriptorCheck fileDescriptorCheck
|
||||
= Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
|
||||
return Collections.singletonList(fileDescriptorCheck);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encapsulates a limit check
|
||||
*/
|
||||
interface Check {
|
||||
|
||||
/**
|
||||
* test if the node fails the check
|
||||
*
|
||||
* @return true if the node failed the check
|
||||
*/
|
||||
boolean check();
|
||||
|
||||
/**
|
||||
* the message for a failed check
|
||||
*
|
||||
* @return the error message on check failure
|
||||
*/
|
||||
String errorMessage();
|
||||
|
||||
}
|
||||
|
||||
static class OsXFileDescriptorCheck extends FileDescriptorCheck {
|
||||
|
||||
public OsXFileDescriptorCheck() {
|
||||
// see constant OPEN_MAX defined in
|
||||
// /usr/include/sys/syslimits.h on OS X and its use in JVM
|
||||
// initialization in int os:init_2(void) defined in the JVM
|
||||
// code for BSD (contains OS X)
|
||||
super(10240);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// visible for testing
|
||||
static class FileDescriptorCheck implements Check {
|
||||
|
||||
private final int limit;
|
||||
|
||||
FileDescriptorCheck() {
|
||||
this(1 << 16);
|
||||
}
|
||||
|
||||
protected FileDescriptorCheck(int limit) {
|
||||
if (limit <= 0) {
|
||||
throw new IllegalArgumentException("limit must be positive but was [" + limit + "]");
|
||||
}
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public final boolean check() {
|
||||
final long maxFileDescriptorCount = getMaxFileDescriptorCount();
|
||||
return maxFileDescriptorCount != -1 && maxFileDescriptorCount < limit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final String errorMessage() {
|
||||
return String.format(
|
||||
Locale.ROOT,
|
||||
"max file descriptors [%d] for elasticsearch process likely too low, increase to at least [%d]",
|
||||
getMaxFileDescriptorCount(),
|
||||
limit
|
||||
);
|
||||
}
|
||||
|
||||
// visible for testing
|
||||
long getMaxFileDescriptorCount() {
|
||||
return ProcessProbe.getInstance().getMaxFileDescriptorCount();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.bootstrap;
|
||||
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
|
||||
public class BootstrapCheckTests extends ESTestCase {
|
||||
|
||||
public void testNonProductionMode() {
|
||||
// nothing should happen since we are in non-production mode
|
||||
BootstrapCheck.check(Settings.EMPTY);
|
||||
}
|
||||
|
||||
public void testFileDescriptorLimits() {
|
||||
final boolean osX = randomBoolean(); // simulates OS X versus non-OS X
|
||||
final int limit = osX ? 10240 : 1 << 16;
|
||||
final AtomicLong maxFileDescriptorCount = new AtomicLong(randomIntBetween(1, limit - 1));
|
||||
final BootstrapCheck.FileDescriptorCheck check;
|
||||
if (osX) {
|
||||
check = new BootstrapCheck.OsXFileDescriptorCheck() {
|
||||
@Override
|
||||
long getMaxFileDescriptorCount() {
|
||||
return maxFileDescriptorCount.get();
|
||||
}
|
||||
};
|
||||
} else {
|
||||
check = new BootstrapCheck.FileDescriptorCheck() {
|
||||
@Override
|
||||
long getMaxFileDescriptorCount() {
|
||||
return maxFileDescriptorCount.get();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
BootstrapCheck.check(true, Collections.singletonList(check));
|
||||
fail("should have failed due to max file descriptors too low");
|
||||
} catch (RuntimeException e) {
|
||||
assertThat(e.getMessage(), containsString("max file descriptors"));
|
||||
}
|
||||
|
||||
maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
|
||||
|
||||
BootstrapCheck.check(true, Collections.singletonList(check));
|
||||
|
||||
// nothing should happen if current file descriptor count is
|
||||
// not available
|
||||
maxFileDescriptorCount.set(-1);
|
||||
BootstrapCheck.check(true, Collections.singletonList(check));
|
||||
}
|
||||
|
||||
public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
|
||||
final IllegalArgumentException e =
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE)));
|
||||
assertThat(e.getMessage(), containsString("limit must be positive but was"));
|
||||
}
|
||||
|
||||
public void testEnforceLimits() {
|
||||
final Set<Setting> enforceSettings = BootstrapCheck.enforceSettings();
|
||||
final Setting setting = randomFrom(Arrays.asList(enforceSettings.toArray(new Setting[enforceSettings.size()])));
|
||||
final Settings settings = Settings.builder().put(setting.getKey(), randomAsciiOfLength(8)).build();
|
||||
assertTrue(BootstrapCheck.enforceLimits(settings));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -19,9 +19,7 @@
|
|||
|
||||
package org.elasticsearch.bootstrap;
|
||||
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.monitor.process.ProcessProbe;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
public class BootstrapSettingsTests extends ESTestCase {
|
||||
|
|
@ -33,22 +31,4 @@ public class BootstrapSettingsTests extends ESTestCase {
|
|||
assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY));
|
||||
}
|
||||
|
||||
@AwaitsFix(bugUrl = "this feature is disabled for snapshot builds, for now - see #16835")
|
||||
public void testEnforceMaxFileDescriptorLimits() {
|
||||
// nothing should happen since we are in OOB mode
|
||||
Bootstrap.enforceOrLogLimits(Settings.EMPTY);
|
||||
|
||||
Settings build = Settings.builder().put(randomFrom(Bootstrap.ENFORCE_SETTINGS.toArray(new Setting[0])).getKey(),
|
||||
"127.0.0.1").build();
|
||||
long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount();
|
||||
try {
|
||||
Bootstrap.enforceOrLogLimits(build);
|
||||
if (maxFileDescriptorCount != -1 && maxFileDescriptorCount < (1 << 16)) {
|
||||
fail("must have enforced limits: " + maxFileDescriptorCount);
|
||||
}
|
||||
} catch (IllegalStateException ex) {
|
||||
assertTrue(ex.getMessage(), ex.getMessage().startsWith("max file descriptors"));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue