diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index 83d4c3419ef..eb2eb14b66f 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -47,7 +47,7 @@ import static org.hamcrest.core.Is.is; // These tests are here today so they have access to a proper REST client. They cannot be in :server:integTest since the REST client needs a // proper transport implementation, and they cannot be REST tests today since they need to restart nodes. When #35599 and friends land we // should be able to move these tests to run against a proper cluster instead. TODO do this. -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoManageMasterNodes = false) public class Zen2RestApiIT extends ESNetty4IntegTestCase { @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java index 7cfc2ea1f28..0a0c1b66d8c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java @@ -31,7 +31,7 @@ import java.io.IOException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; @ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0, - autoMinMasterNodes = false) + autoManageMasterNodes = false) public class IndicesExistsIT extends ESIntegTestCase { public void testIndexExistsWithBlocksInPlace() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java index 1317183f286..45d8f4c8c0b 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java @@ -39,7 +39,7 @@ import java.util.concurrent.CyclicBarrier; import static org.hamcrest.Matchers.equalTo; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class IndexingMasterFailoverIT extends ESIntegTestCase { @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index cb1443bdf37..164c74423aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -62,7 +62,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.cluster.coordination:TRACE") public class MinimumMasterNodesIT extends ESIntegTestCase { diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index f80a5befa83..d8be488f5d4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -39,7 +39,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) @TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE") public class SpecificMasterNodesIT extends ESIntegTestCase { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index b2ed28dcdaa..c7373401913 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -48,7 +48,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.cluster.coordination:TRACE") public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase { diff --git a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java index 5d8e3407e18..ac6a2189092 100644 --- a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java @@ -50,7 +50,7 @@ import static org.hamcrest.Matchers.not; numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false, - autoMinMasterNodes = false) + autoManageMasterNodes = false) public class SingleNodeDiscoveryIT extends ESIntegTestCase { @Override diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index 86976d553fa..84188f80aae 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -34,7 +34,7 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; -@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class RecoverAfterNodesIT extends ESIntegTestCase { private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 365687db346..21e7dc3f683 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1671,10 +1671,11 @@ public abstract class ESIntegTestCase extends ESTestCase { boolean supportsDedicatedMasters() default true; /** - * The cluster automatically manages the {@link ElectMasterService#DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING} by default - * as nodes are started and stopped. Set this to false to manage the setting manually. + * Indicates whether the cluster automatically manages cluster bootstrapping and the removal of any master-eligible nodes as well + * as {@link ElectMasterService#DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING} if running the pre-7.0 cluster coordination + * implementation. If set to {@code false} then the tests must manage these things explicitly. */ - boolean autoMinMasterNodes() default true; + boolean autoManageMasterNodes() default true; /** * Returns the number of client nodes in the cluster. Default is {@link InternalTestCluster#DEFAULT_NUM_CLIENT_NODES}, a @@ -1768,9 +1769,9 @@ public abstract class ESIntegTestCase extends ESTestCase { return annotation == null ? true : annotation.supportsDedicatedMasters(); } - private boolean getAutoMinMasterNodes() { + private boolean getAutoManageMasterNodes() { ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class); - return annotation == null ? true : annotation.autoMinMasterNodes(); + return annotation == null ? true : annotation.autoManageMasterNodes(); } private int getNumDataNodes() { @@ -1920,7 +1921,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } mockPlugins = mocks; } - return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoMinMasterNodes(), + return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoManageMasterNodes(), minNumDataNodes, maxNumDataNodes, InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(), nodePrefix, mockPlugins, getClientWrapper(), forbidPrivateIndexSettings()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 60061877754..66208479e06 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -238,7 +238,7 @@ public final class InternalTestCluster extends TestCluster { private final ExecutorService executor; - private final boolean autoManageMinMasterNodes; + private final boolean autoManageMasterNodes; private final Collection> mockPlugins; @@ -261,7 +261,7 @@ public final class InternalTestCluster extends TestCluster { final long clusterSeed, final Path baseDir, final boolean randomlyAddDedicatedMasters, - final boolean autoManageMinMasterNodes, + final boolean autoManageMasterNodes, final int minNumDataNodes, final int maxNumDataNodes, final String clusterName, @@ -274,7 +274,7 @@ public final class InternalTestCluster extends TestCluster { clusterSeed, baseDir, randomlyAddDedicatedMasters, - autoManageMinMasterNodes, + autoManageMasterNodes, minNumDataNodes, maxNumDataNodes, clusterName, @@ -290,7 +290,7 @@ public final class InternalTestCluster extends TestCluster { final long clusterSeed, final Path baseDir, final boolean randomlyAddDedicatedMasters, - final boolean autoManageMinMasterNodes, + final boolean autoManageMasterNodes, final int minNumDataNodes, final int maxNumDataNodes, final String clusterName, @@ -301,7 +301,7 @@ public final class InternalTestCluster extends TestCluster { final Function clientWrapper, final boolean forbidPrivateIndexSettings) { super(clusterSeed); - this.autoManageMinMasterNodes = autoManageMinMasterNodes; + this.autoManageMasterNodes = autoManageMasterNodes; this.clientWrapper = clientWrapper; this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; this.baseDir = baseDir; @@ -359,7 +359,7 @@ public final class InternalTestCluster extends TestCluster { "[{}] (data) nodes and [{}] coord only nodes (min_master_nodes are [{}])", clusterName, SeedUtils.formatSeed(clusterSeed), numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes, - autoManageMinMasterNodes ? "auto-managed" : "manual"); + autoManageMasterNodes ? "auto-managed" : "manual"); this.nodeConfigurationSource = nodeConfigurationSource; numDataPaths = random.nextInt(5) == 0 ? 2 + random.nextInt(3) : 1; Builder builder = Settings.builder(); @@ -409,12 +409,11 @@ public final class InternalTestCluster extends TestCluster { /** * Sets {@link #bootstrapMasterNodeIndex} to the given value, see {@link #bootstrapMasterNodeWithSpecifiedIndex(List)} * for the description of how this field is used. - * It's only possible to change {@link #bootstrapMasterNodeIndex} value if autoManageMinMasterNodes is false. + * It's only possible to change {@link #bootstrapMasterNodeIndex} value if autoManageMasterNodes is false. */ public void setBootstrapMasterNodeIndex(int bootstrapMasterNodeIndex) { - if (autoManageMinMasterNodes && bootstrapMasterNodeIndex != -1) { - throw new AssertionError("bootstrapMasterNodeIndex should be -1 if autoManageMinMasterNodes is true"); - } + assert autoManageMasterNodes == false || bootstrapMasterNodeIndex == -1 + : "bootstrapMasterNodeIndex should be -1 if autoManageMasterNodes is true, but was " + bootstrapMasterNodeIndex; this.bootstrapMasterNodeIndex = bootstrapMasterNodeIndex; } @@ -425,7 +424,7 @@ public final class InternalTestCluster extends TestCluster { /** returns true if the {@link ElectMasterService#DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING} setting is auto managed by this cluster */ public boolean getAutoManageMinMasterNode() { - return autoManageMinMasterNodes; + return autoManageMasterNodes; } public String[] getNodeNames() { @@ -653,10 +652,10 @@ public final class InternalTestCluster extends TestCluster { final boolean usingSingleNodeDiscovery = discoveryType.equals("single-node"); final boolean usingZen1 = usingZen1(updatedSettings.build()); if (usingSingleNodeDiscovery == false) { - if (autoManageMinMasterNodes) { - assertThat("min master nodes may not be set when auto managed", + if (autoManageMasterNodes) { + assertThat("min master nodes may not be set when master nodes are auto managed", updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), nullValue()); - assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting", + assertThat("if master nodes are automatically managed then nodes must complete a join cycle when starting", updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue()); if (usingZen1) { @@ -1135,7 +1134,7 @@ public final class InternalTestCluster extends TestCluster { if (wipeData) { wipePendingDataDirectories(); } - if (nodes.size() > 0 && autoManageMinMasterNodes) { + if (nodes.size() > 0 && autoManageMasterNodes) { updateMinMasterNodes(getMasterNodesCount()); } logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", @@ -1169,7 +1168,7 @@ public final class InternalTestCluster extends TestCluster { assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; final int numberOfMasterNodes = numSharedDedicatedMasterNodes > 0 ? numSharedDedicatedMasterNodes : numSharedDataNodes; final int defaultMinMasterNodes = (numberOfMasterNodes / 2) + 1; - final List toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go due to min master nodes + final List toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go final Runnable onTransportServiceStarted = () -> rebuildUnicastHostFiles(toStartAndPublish); final List settings = new ArrayList<>(); @@ -1202,7 +1201,7 @@ public final class InternalTestCluster extends TestCluster { .map(Node.NODE_NAME_SETTING::get) .collect(Collectors.toList()); - if (prevNodeCount == 0 && autoManageMinMasterNodes) { + if (prevNodeCount == 0 && autoManageMasterNodes) { if (numSharedDedicatedMasterNodes > 0) { autoBootstrapMasterNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDedicatedMasterNodes - 1); } else if (numSharedDataNodes > 0) { @@ -1225,7 +1224,7 @@ public final class InternalTestCluster extends TestCluster { nextNodeId.set(newSize); assert size() == newSize; - if (autoManageMinMasterNodes && newSize > 0) { + if (autoManageMasterNodes && newSize > 0) { validateClusterFormed(); } logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", @@ -1670,7 +1669,7 @@ public final class InternalTestCluster extends TestCluster { .filter(nac -> nodes.containsKey(nac.name) == false) // filter out old masters .count(); final int currentMasters = getMasterNodesCount(); - if (autoManageMinMasterNodes && currentMasters > 0 && newMasters > 0 && + if (autoManageMasterNodes && currentMasters > 0 && newMasters > 0 && getMinMasterNodes(currentMasters + newMasters) <= currentMasters) { // if we're adding too many master-eligible nodes at once, we can't update the min master setting before adding the nodes. updateMinMasterNodes(currentMasters + newMasters); @@ -1689,7 +1688,7 @@ public final class InternalTestCluster extends TestCluster { } nodeAndClients.forEach(this::publishNode); - if (autoManageMinMasterNodes && currentMasters > 0 && newMasters > 0 && + if (autoManageMasterNodes && currentMasters > 0 && newMasters > 0 && getMinMasterNodes(currentMasters + newMasters) > currentMasters) { // update once masters have joined validateClusterFormed(); @@ -1802,7 +1801,7 @@ public final class InternalTestCluster extends TestCluster { Set excludedNodeIds = excludeMasters(Collections.singleton(nodeAndClient)); final Settings newSettings = nodeAndClient.closeForRestart(callback, - autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1); + autoManageMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1); removeExclusions(excludedNodeIds); @@ -1822,10 +1821,8 @@ public final class InternalTestCluster extends TestCluster { } if (callback.validateClusterForming() || excludedNodeIds.isEmpty() == false) { - // we have to validate cluster size if updateMinMaster == true, because we need the - // second node to join in order to increment min_master_nodes back to 2. - // we also have to do via the node that was just restarted as it may be that the master didn't yet process - // the fact it left + // we have to validate cluster size to ensure that the restarted node has rejoined the cluster if it was master-eligible; + // we have to do this via the node that was just restarted as it may be that the master didn't yet process the fact that it left validateClusterFormed(nodeAndClient.name); } @@ -1845,7 +1842,7 @@ public final class InternalTestCluster extends TestCluster { private Set excludeMasters(Collection nodeAndClients) { assert Thread.holdsLock(this); final Set excludedNodeIds = new HashSet<>(); - if (autoManageMinMasterNodes && nodeAndClients.size() > 0) { + if (autoManageMasterNodes && nodeAndClients.size() > 0) { final long currentMasters = nodes.values().stream().filter(NodeAndClient::isMasterEligible).count(); final long stoppingMasters = nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).count(); @@ -1896,7 +1893,7 @@ public final class InternalTestCluster extends TestCluster { final Settings[] newNodeSettings = new Settings[nextNodeId.get()]; Map, List> nodesByRoles = new HashMap<>(); Set[] rolesOrderedByOriginalStartupOrder = new Set[nextNodeId.get()]; - final int minMasterNodes = autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1; + final int minMasterNodes = autoManageMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1; for (NodeAndClient nodeAndClient : nodes.values()) { callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient()); logger.info("Stopping and resetting node [{}] ", nodeAndClient.name); @@ -2090,7 +2087,7 @@ public final class InternalTestCluster extends TestCluster { public synchronized List startNodes(Settings... extraSettings) { final int newMasterCount = Math.toIntExact(Stream.of(extraSettings).filter(Node.NODE_MASTER_SETTING::get).count()); final int defaultMinMasterNodes; - if (autoManageMinMasterNodes) { + if (autoManageMasterNodes) { defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + newMasterCount); } else { defaultMinMasterNodes = -1; @@ -2098,7 +2095,7 @@ public final class InternalTestCluster extends TestCluster { final List nodes = new ArrayList<>(); final int prevMasterCount = getMasterNodesCount(); int autoBootstrapMasterNodeIndex = - prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(extraSettings) + prevMasterCount == 0 && autoManageMasterNodes && newMasterCount > 0 && Arrays.stream(extraSettings) .allMatch(s -> Node.NODE_MASTER_SETTING.get(s) == false || ZEN2_DISCOVERY_TYPE.equals(DISCOVERY_TYPE_SETTING.get(s))) ? RandomNumbers.randomIntBetween(random, 0, newMasterCount - 1) : -1; @@ -2133,7 +2130,7 @@ public final class InternalTestCluster extends TestCluster { nodes.add(nodeAndClient); } startAndPublishNodesAndClients(nodes); - if (autoManageMinMasterNodes) { + if (autoManageMasterNodes) { validateClusterFormed(); } return nodes.stream().map(NodeAndClient::getName).collect(Collectors.toList()); @@ -2165,7 +2162,7 @@ public final class InternalTestCluster extends TestCluster { * @param eligibleMasterNodeCount the number of master eligible nodes to use as basis for the min master node setting */ private void updateMinMasterNodes(int eligibleMasterNodeCount) { - assert autoManageMinMasterNodes; + assert autoManageMasterNodes; final int minMasterNodes = getMinMasterNodes(eligibleMasterNodeCount); if (getMasterNodesCount() > 0) { // there should be at least one master to update diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java index a2f6b3ed654..a018bc16631 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java +++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java @@ -37,7 +37,7 @@ import java.util.Set; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) public class NetworkDisruptionIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() {