Disc: Move AbstractDisruptionTC to filebased D. (#34461)

* Discovery: Move AbstractDisruptionTestCase to file-based discovery.
* Relates #33675
* Simplify away ClusterDiscoveryConfiguration
This commit is contained in:
Armin Braun 2018-10-16 15:28:40 +01:00 committed by GitHub
parent af2877cb7b
commit ea576a8ca2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 61 additions and 244 deletions

View File

@ -19,6 +19,7 @@
package org.elasticsearch.discovery;
import java.nio.file.Path;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
@ -33,7 +34,8 @@ import org.elasticsearch.discovery.zen.ZenPing;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.NodeConfigurationSource;
import org.elasticsearch.test.discovery.TestZenDiscovery;
import org.elasticsearch.test.disruption.NetworkDisruption;
import org.elasticsearch.test.disruption.NetworkDisruption.Bridge;
@ -52,9 +54,9 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
@ -62,7 +64,7 @@ public abstract class AbstractDisruptionTestCase extends ESIntegTestCase {
static final TimeValue DISRUPTION_HEALING_OVERHEAD = TimeValue.timeValueSeconds(40); // we use 30s as timeout in many places.
private ClusterDiscoveryConfiguration discoveryConfig;
private NodeConfigurationSource discoveryConfig;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -116,18 +118,14 @@ public abstract class AbstractDisruptionTestCase extends ESIntegTestCase {
}
}
List<String> startCluster(int numberOfNodes) throws ExecutionException, InterruptedException {
List<String> startCluster(int numberOfNodes) {
return startCluster(numberOfNodes, -1);
}
List<String> startCluster(int numberOfNodes, int minimumMasterNode) throws ExecutionException, InterruptedException {
return startCluster(numberOfNodes, minimumMasterNode, null);
}
List<String> startCluster(int numberOfNodes, int minimumMasterNode, @Nullable int[] unicastHostsOrdinals) throws
ExecutionException, InterruptedException {
configureCluster(numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
List<String> nodes = internalCluster().startNodes(numberOfNodes);
List<String> startCluster(int numberOfNodes, int minimumMasterNode) {
configureCluster(numberOfNodes, minimumMasterNode);
InternalTestCluster internalCluster = internalCluster();
List<String> nodes = internalCluster.startNodes(numberOfNodes);
ensureStableCluster(numberOfNodes);
// TODO: this is a temporary solution so that nodes will not base their reaction to a partition based on previous successful results
@ -154,20 +152,11 @@ public abstract class AbstractDisruptionTestCase extends ESIntegTestCase {
return Arrays.asList(MockTransportService.TestPlugin.class);
}
void configureCluster(
int numberOfNodes,
@Nullable int[] unicastHostsOrdinals,
int minimumMasterNode
) throws ExecutionException, InterruptedException {
configureCluster(DEFAULT_SETTINGS, numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
void configureCluster(int numberOfNodes, int minimumMasterNode) {
configureCluster(DEFAULT_SETTINGS, numberOfNodes, minimumMasterNode);
}
void configureCluster(
Settings settings,
int numberOfNodes,
@Nullable int[] unicastHostsOrdinals,
int minimumMasterNode
) throws ExecutionException, InterruptedException {
void configureCluster(Settings settings, int numberOfNodes, int minimumMasterNode) {
if (minimumMasterNode < 0) {
minimumMasterNode = numberOfNodes / 2 + 1;
}
@ -177,14 +166,21 @@ public abstract class AbstractDisruptionTestCase extends ESIntegTestCase {
.put(settings)
.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numberOfNodes)
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
.putList(DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file")
.build();
if (discoveryConfig == null) {
if (unicastHostsOrdinals == null) {
discoveryConfig = new ClusterDiscoveryConfiguration.UnicastZen(numberOfNodes, nodeSettings);
} else {
discoveryConfig = new ClusterDiscoveryConfiguration.UnicastZen(numberOfNodes, nodeSettings, unicastHostsOrdinals);
discoveryConfig = new NodeConfigurationSource() {
@Override
public Settings nodeSettings(final int nodeOrdinal) {
return nodeSettings;
}
@Override
public Path nodeConfigPath(final int nodeOrdinal) {
return null;
}
};
}
}

View File

@ -363,7 +363,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
*/
public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Exception {
// don't use DEFAULT settings (which can cause node disconnects on a slow CI machine)
configureCluster(Settings.EMPTY, 3, null, 1);
configureCluster(Settings.EMPTY, 3, 1);
internalCluster().startMasterOnlyNode();
final String node_1 = internalCluster().startDataOnlyNode();
@ -390,7 +390,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception {
// test for https://github.com/elastic/elasticsearch/issues/8823
configureCluster(2, null, 1);
configureCluster(2, 1);
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
internalCluster().startDataOnlyNode(Settings.EMPTY);
@ -421,7 +421,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
.put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
.build();
final String idxName = "test";
configureCluster(settings, 3, null, 2);
configureCluster(settings, 3, 2);
final List<String> allMasterEligibleNodes = internalCluster().startMasterOnlyNodes(2);
final String dataNode = internalCluster().startDataOnlyNode();
ensureStableCluster(3);

View File

@ -59,7 +59,8 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
public void testIsolatedUnicastNodes() throws Exception {
List<String> nodes = startCluster(4, -1, new int[]{0});
internalCluster().setHostsListContainsOnlyFirstNode(true);
List<String> nodes = startCluster(4, -1);
// Figure out what is the elected master node
final String unicastTarget = nodes.get(0);
@ -98,7 +99,8 @@ public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
* The rejoining node should take this master node and connect.
*/
public void testUnicastSinglePingResponseContainsMaster() throws Exception {
List<String> nodes = startCluster(4, -1, new int[]{0});
internalCluster().setHostsListContainsOnlyFirstNode(true);
List<String> nodes = startCluster(4, -1);
// Figure out what is the elected master node
final String masterNode = internalCluster().getMasterName();
logger.info("---> legit elected master node={}", masterNode);
@ -194,7 +196,7 @@ public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
}
public void testClusterFormingWithASlowNode() throws Exception {
configureCluster(3, null, 2);
configureCluster(3, 2);
SlowClusterStateProcessing disruption = new SlowClusterStateProcessing(random(), 0, 0, 1000, 2000);
@ -210,7 +212,7 @@ public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
}
public void testElectMasterWithLatestVersion() throws Exception {
configureCluster(3, null, 2);
configureCluster(3, 2);
final Set<String> nodes = new HashSet<>(internalCluster().startNodes(3));
ensureStableCluster(3);
ServiceDisruptionScheme isolateAllNodes =

View File

@ -59,7 +59,7 @@ public class SnapshotDisruptionIT extends AbstractDisruptionTestCase {
.put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
.build();
final String idxName = "test";
configureCluster(settings, 4, null, 2);
configureCluster(settings, 4, 2);
final List<String> allMasterEligibleNodes = internalCluster().startMasterOnlyNodes(3);
final String dataNode = internalCluster().startDataOnlyNode();
ensureStableCluster(4);

View File

@ -231,6 +231,9 @@ public final class InternalTestCluster extends TestCluster {
private ServiceDisruptionScheme activeDisruptionScheme;
private Function<Client, Client> clientWrapper;
// If set to true only the first node in the cluster will be made a unicast node
private boolean hostsListContainsOnlyFirstNode;
public InternalTestCluster(
final long clusterSeed,
final Path baseDir,
@ -1605,12 +1608,17 @@ public final class InternalTestCluster extends TestCluster {
private final Object discoveryFileMutex = new Object();
private void rebuildUnicastHostFiles(Collection<NodeAndClient> newNodes) {
private void rebuildUnicastHostFiles(List<NodeAndClient> newNodes) {
// cannot be a synchronized method since it's called on other threads from within synchronized startAndPublishNodesAndClients()
synchronized (discoveryFileMutex) {
try {
List<String> discoveryFileContents = Stream.concat(nodes.values().stream(), newNodes.stream())
.map(nac -> nac.node.injector().getInstance(TransportService.class)).filter(Objects::nonNull)
Stream<NodeAndClient> unicastHosts = Stream.concat(nodes.values().stream(), newNodes.stream());
if (hostsListContainsOnlyFirstNode) {
unicastHosts = unicastHosts.limit(1L);
}
List<String> discoveryFileContents = unicastHosts.map(
nac -> nac.node.injector().getInstance(TransportService.class)
).filter(Objects::nonNull)
.map(TransportService::getLocalNode).filter(Objects::nonNull).filter(DiscoveryNode::isMasterNode)
.map(n -> n.getAddress().toString())
.distinct().collect(Collectors.toList());
@ -2038,6 +2046,9 @@ public final class InternalTestCluster extends TestCluster {
return filterNodes(nodes, NodeAndClient::isMasterEligible).size();
}
public void setHostsListContainsOnlyFirstNode(boolean hostsListContainsOnlyFirstNode) {
this.hostsListContainsOnlyFirstNode = hostsListContainsOnlyFirstNode;
}
public void setDisruptionScheme(ServiceDisruptionScheme scheme) {
assert activeDisruptionScheme == null :

View File

@ -1,185 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.discovery;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.SysGlobals;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.network.NetworkUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.mocksocket.MockServerSocket;
import org.elasticsearch.test.NodeConfigurationSource;
import org.elasticsearch.transport.TcpTransport;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
public class ClusterDiscoveryConfiguration extends NodeConfigurationSource {
/**
* The number of ports in the range used for this JVM
*/
private static final int PORTS_PER_JVM = 100;
private static final int JVM_ORDINAL = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0"));
/**
* a per-JVM unique offset to be used for calculating unique port ranges.
*/
private static final int JVM_BASE_PORT_OFFSET = PORTS_PER_JVM * (JVM_ORDINAL + 1);
static Settings DEFAULT_NODE_SETTINGS = Settings.EMPTY;
private static final String IP_ADDR = "127.0.0.1";
final int numOfNodes;
final Settings nodeSettings;
final Settings transportClientSettings;
public ClusterDiscoveryConfiguration(int numOfNodes, Settings extraSettings) {
this.numOfNodes = numOfNodes;
this.nodeSettings = Settings.builder().put(DEFAULT_NODE_SETTINGS).put(extraSettings).build();
this.transportClientSettings = Settings.builder().put(extraSettings).build();
}
@Override
public Settings nodeSettings(int nodeOrdinal) {
return nodeSettings;
}
@Override
public Path nodeConfigPath(int nodeOrdinal) {
return null;
}
@Override
public Settings transportClientSettings() {
return transportClientSettings;
}
public static class UnicastZen extends ClusterDiscoveryConfiguration {
// this variable is incremented on each bind attempt and will maintain the next port that should be tried
private static int nextPort = calcBasePort();
private final int[] unicastHostOrdinals;
private final int[] unicastHostPorts;
public UnicastZen(int numOfNodes, Settings extraSettings) {
this(numOfNodes, numOfNodes, extraSettings);
}
public UnicastZen(int numOfNodes, int numOfUnicastHosts, Settings extraSettings) {
super(numOfNodes, extraSettings);
if (numOfUnicastHosts == numOfNodes) {
unicastHostOrdinals = new int[numOfNodes];
for (int i = 0; i < numOfNodes; i++) {
unicastHostOrdinals[i] = i;
}
} else {
Set<Integer> ordinals = new HashSet<>(numOfUnicastHosts);
while (ordinals.size() != numOfUnicastHosts) {
ordinals.add(RandomizedTest.randomInt(numOfNodes - 1));
}
unicastHostOrdinals = CollectionUtils.toArray(ordinals);
}
this.unicastHostPorts = unicastHostPorts(numOfNodes);
assert unicastHostOrdinals.length <= unicastHostPorts.length;
}
public UnicastZen(int numOfNodes, int[] unicastHostOrdinals) {
this(numOfNodes, Settings.EMPTY, unicastHostOrdinals);
}
public UnicastZen(int numOfNodes, Settings extraSettings, int[] unicastHostOrdinals) {
super(numOfNodes, extraSettings);
this.unicastHostOrdinals = unicastHostOrdinals;
this.unicastHostPorts = unicastHostPorts(numOfNodes);
assert unicastHostOrdinals.length <= unicastHostPorts.length;
}
private static int calcBasePort() {
return 30000 + JVM_BASE_PORT_OFFSET;
}
@Override
public Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder().put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numOfNodes);
String[] unicastHosts = new String[unicastHostOrdinals.length];
if (nodeOrdinal >= unicastHostPorts.length) {
throw new ElasticsearchException("nodeOrdinal [" + nodeOrdinal + "] is greater than the number unicast ports ["
+ unicastHostPorts.length + "]");
} else {
// we need to pin the node port & host so we'd know where to point things
builder.put(TcpTransport.PORT.getKey(), unicastHostPorts[nodeOrdinal]);
builder.put(TcpTransport.HOST.getKey(), IP_ADDR); // only bind on one IF we use v4 here by default
for (int i = 0; i < unicastHostOrdinals.length; i++) {
unicastHosts[i] = IP_ADDR + ":" + (unicastHostPorts[unicastHostOrdinals[i]]);
}
}
builder.putList("discovery.zen.ping.unicast.hosts", unicastHosts);
return builder.put(super.nodeSettings(nodeOrdinal)).build();
}
@SuppressForbidden(reason = "we know we pass a IP address")
protected static synchronized int[] unicastHostPorts(int numHosts) {
int[] unicastHostPorts = new int[numHosts];
final int basePort = calcBasePort();
final int maxPort = basePort + PORTS_PER_JVM;
int tries = 0;
for (int i = 0; i < unicastHostPorts.length; i++) {
boolean foundPortInRange = false;
while (tries < PORTS_PER_JVM && !foundPortInRange) {
try (ServerSocket serverSocket = new MockServerSocket()) {
// Set SO_REUSEADDR as we may bind here and not be able to reuse the address immediately without it.
serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress());
serverSocket.bind(new InetSocketAddress(IP_ADDR, nextPort));
// bind was a success
foundPortInRange = true;
unicastHostPorts[i] = nextPort;
} catch (IOException e) {
// Do nothing
}
nextPort++;
if (nextPort >= maxPort) {
// Roll back to the beginning of the range and do not go into another JVM's port range
nextPort = basePort;
}
tries++;
}
if (!foundPortInRange) {
throw new ElasticsearchException("could not find enough open ports in range [" + basePort + "-" + maxPort
+ "]. required [" + unicastHostPorts.length + "] ports");
}
}
return unicastHostPorts;
}
}
}

View File

@ -152,7 +152,7 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
public static void initDefaultSettings() {
if (SECURITY_DEFAULT_SETTINGS == null) {
SECURITY_DEFAULT_SETTINGS =
new SecuritySettingsSource(defaultMaxNumberOfNodes(), randomBoolean(), createTempDir(), Scope.SUITE);
new SecuritySettingsSource(randomBoolean(), createTempDir(), Scope.SUITE);
}
}
@ -367,7 +367,7 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
private class CustomSecuritySettingsSource extends SecuritySettingsSource {
private CustomSecuritySettingsSource(boolean sslEnabled, Path configDir, Scope scope) {
super(maxNumberOfNodes(), sslEnabled, configDir, scope);
super(sslEnabled, configDir, scope);
}
@Override

View File

@ -17,7 +17,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
import org.elasticsearch.transport.Netty4Plugin;
import org.elasticsearch.xpack.core.XPackClientPlugin;
import org.elasticsearch.xpack.core.XPackSettings;
@ -49,13 +48,10 @@ import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
/**
* {@link org.elasticsearch.test.NodeConfigurationSource} subclass that allows to set all needed settings for x-pack security.
* Unicast discovery is configured through {@link org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration.UnicastZen},
* also x-pack is installed with all the needed configuration and files.
* X-pack is installed with all the needed configuration and files.
* To avoid conflicts, every cluster should have its own instance of this class as some configuration files need to be created.
*/
public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.UnicastZen {
public static final Settings DEFAULT_SETTINGS = Settings.EMPTY;
public class SecuritySettingsSource extends NodeConfigurationSource {
public static final String TEST_USER_NAME = "test_user";
public static final String TEST_PASSWORD_HASHED =
@ -93,13 +89,11 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
/**
* Creates a new {@link org.elasticsearch.test.NodeConfigurationSource} for the security configuration.
*
* @param numOfNodes the number of nodes for proper unicast configuration (can be more than actually available)
* @param sslEnabled whether ssl is enabled
* @param parentFolder the parent folder that will contain all of the configuration files that need to be created
* @param scope the scope of the test that is requiring an instance of SecuritySettingsSource
*/
public SecuritySettingsSource(int numOfNodes, boolean sslEnabled, Path parentFolder, Scope scope) {
super(numOfNodes, DEFAULT_SETTINGS);
public SecuritySettingsSource(boolean sslEnabled, Path parentFolder, Scope scope) {
this.parentFolder = parentFolder;
this.subfolderPrefix = scope.name();
this.sslEnabled = sslEnabled;
@ -129,7 +123,7 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
writeFile(xpackConf, "users", configUsers());
writeFile(xpackConf, "users_roles", configUsersRoles());
Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal))
Settings.Builder builder = Settings.builder()
.put(XPackSettings.SECURITY_ENABLED.getKey(), true)
.put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
.put(NetworkModule.HTTP_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
@ -156,10 +150,9 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
@Override
public Settings transportClientSettings() {
Settings superSettings = super.transportClientSettings();
Settings.Builder builder = Settings.builder().put(superSettings);
Settings.Builder builder = Settings.builder();
addClientSSLSettings(builder, "");
addDefaultSecurityTransportType(builder, superSettings);
addDefaultSecurityTransportType(builder, Settings.EMPTY);
if (randomBoolean()) {
builder.put(SecurityField.USER_SETTING.getKey(),

View File

@ -65,7 +65,7 @@ public abstract class SecuritySingleNodeTestCase extends ESSingleNodeTestCase {
public static void initDefaultSettings() {
if (SECURITY_DEFAULT_SETTINGS == null) {
SECURITY_DEFAULT_SETTINGS =
new SecuritySettingsSource(1, randomBoolean(), createTempDir(), ESIntegTestCase.Scope.SUITE);
new SecuritySettingsSource(randomBoolean(), createTempDir(), ESIntegTestCase.Scope.SUITE);
}
}
@ -235,7 +235,7 @@ public abstract class SecuritySingleNodeTestCase extends ESSingleNodeTestCase {
private class CustomSecuritySettingsSource extends SecuritySettingsSource {
private CustomSecuritySettingsSource(boolean sslEnabled, Path configDir, ESIntegTestCase.Scope scope) {
super(1, sslEnabled, configDir, scope);
super(sslEnabled, configDir, scope);
}
@Override

View File

@ -178,7 +178,7 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
logger.info("--> remote indexing enabled. security enabled: [{}], SSL enabled: [{}], nodes: [{}]", useSecurity, useSSL,
numNodes);
SecuritySettingsSource cluster2SettingsSource =
new SecuritySettingsSource(numNodes, useSSL, createTempDir(), Scope.SUITE) {
new SecuritySettingsSource(useSSL, createTempDir(), Scope.SUITE) {
@Override
public Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()

View File

@ -95,7 +95,7 @@ public class RemoteIndexAuditTrailStartingTests extends SecurityIntegTestCase {
// Setup a second test cluster with a single node, security enabled, and SSL
final int numNodes = 1;
SecuritySettingsSource cluster2SettingsSource =
new SecuritySettingsSource(numNodes, sslEnabled, createTempDir(), Scope.TEST) {
new SecuritySettingsSource(sslEnabled, createTempDir(), Scope.TEST) {
@Override
public Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()