Update tests for new auto management of min master nodes (elastic/elasticsearch#4068)

The internal test cluster now auto manages min_master_nodes  ( see https://github.com/elastic/elasticsearch/pull/21458 ). This requires some code changes but also changes the timings of forming a cluster. This has had a funny side effect where the master is no longer always the first node to be started in the cluster. This caused issues with watcher tests which freeze time.

Original commit: elastic/x-pack-elasticsearch@1e5ea8ae94

elasticsearch/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java

@@ -21,12 +21,11 @@ import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 
-import static org.elasticsearch.license.TestUtils.generateSignedLicense;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.nullValue;
 
-@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0)
+@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false)
 public class LicenseServiceClusterTests extends AbstractLicensesIntegrationTestCase {
 
     @Override

elasticsearch/src/test/java/org/elasticsearch/license/TribeTransportTestCase.java

@@ -46,7 +46,7 @@ import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-@ClusterScope(scope = Scope.TEST, transportClientRatio = 0, numClientNodes = 1, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, transportClientRatio = 0, numClientNodes = 1, numDataNodes = 2)
 public abstract class TribeTransportTestCase extends ESIntegTestCase {
 
     protected List<String> enabledFeatures() {
@@ -99,12 +99,11 @@ public abstract class TribeTransportTestCase extends ESIntegTestCase {
             }
         };
         final InternalTestCluster cluster2 = new InternalTestCluster(
-                randomLong(), createTempDir(), true, 2, 2,
+                randomLong(), createTempDir(), true, true, 2, 2,
                 UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 1, false, "tribe_node2",
                 getMockPlugins(), getClientWrapper());
 
         cluster2.beforeTest(random(), 0.0);
-        cluster2.ensureAtLeastNumDataNodes(2);
 
         logger.info("create 2 indices, test1 on t1, and test2 on t2");
         assertAcked(internalCluster().client().admin().indices().prepareCreate("test1").get());

elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/MultiNodesStatsTests.java

@@ -69,8 +69,10 @@ public class MultiNodesStatsTests extends MonitoringIntegTestCase {
 
         n = randomIntBetween(1, 2);
         logger.debug("--> starting {} extra nodes", n);
-        InternalTestCluster.Async<List<String>> extraNodes = internalCluster().startNodesAsync(n);
+        // starting one by one to allow moving , for example, from a 2 node cluster to a 4 one while updating min_master_nodes
-        extraNodes.get();
+        for (int i=0;i<n;i++) {
+            internalCluster().startNode();
+        }
         nodes += n;
 
         final int nbNodes = nodes;

elasticsearch/src/test/java/org/elasticsearch/xpack/security/SecurityTribeIT.java

@@ -75,7 +75,7 @@ public class SecurityTribeIT extends NativeRealmIntegTestCase {
         if (cluster2 == null) {
             SecuritySettingsSource cluster2SettingsSource =
                     new SecuritySettingsSource(defaultMaxNumberOfNodes(), useSSL, systemKey(), createTempDir(), Scope.SUITE);
-            cluster2 = new InternalTestCluster(randomLong(), createTempDir(), true, 1, 2,
+            cluster2 = new InternalTestCluster(randomLong(), createTempDir(), true, true, 1, 2,
                     UUIDs.randomBase64UUID(random()), cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(),
                     getClientWrapper());
             cluster2.beforeTest(random(), 0.1);

elasticsearch/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java

@@ -153,7 +153,7 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
         if (useSecurity == false) {
             mockPlugins.add(MockTcpTransportPlugin.class);
         }
-        remoteCluster = new InternalTestCluster(randomLong(), createTempDir(), false, numNodes, numNodes, cluster2Name,
+        remoteCluster = new InternalTestCluster(randomLong(), createTempDir(), false, true, numNodes, numNodes, cluster2Name,
                 cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, mockPlugins,
                 useSecurity ? getClientWrapper() : Function.identity());
         remoteCluster.beforeTest(random(), 0.5);

elasticsearch/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java

@@ -110,8 +110,7 @@ public class RemoteIndexAuditTrailStartingTests extends SecurityIntegTestCase {
                 return builder.build();
             }
         };
-        remoteCluster = new InternalTestCluster(randomLong(), createTempDir(), false,
+        remoteCluster = new InternalTestCluster(randomLong(), createTempDir(), false, true, numNodes, numNodes,
-                numNodes, numNodes,
                 cluster2Name, cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(), getClientWrapper());
         remoteCluster.beforeTest(random(), 0.5);
         assertNoTimeout(remoteCluster.client().admin().cluster().prepareHealth().setWaitForGreenStatus().get());

elasticsearch/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -37,6 +38,7 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.TestCluster;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -588,6 +590,12 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
     }
 
     protected void ensureLicenseEnabled() throws Exception {
+        if (timeWarped()) {
+            // the master generates a license which starts now. We have to make sure all nodes
+            // advance their time so that the license will be valid
+            progressClocksAboveMaster(internalCluster());
+        }
+
         assertBusy(() -> {
             for (XPackLicenseState licenseState : internalCluster().getInstances(XPackLicenseState.class)) {
                 assertThat(licenseState.isWatcherAllowed(), is(true));
@@ -595,6 +603,21 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase
         });
     }
 
+    private void progressClocksAboveMaster(InternalTestCluster cluster) {
+        long minClock = Long.MAX_VALUE;
+        long maxClock = Long.MIN_VALUE;
+        for (Clock clock: cluster.getInstances(Clock.class)) {
+            final long millis = clock.millis();
+            minClock = Math.min(millis, minClock);
+            maxClock = Math.max(millis, maxClock);
+        }
+        // now move all the clocks ahead to make sure they are beyond the highest clock
+        final TimeValue delta = TimeValue.timeValueMillis(maxClock - minClock);
+        for (Clock clock: cluster.getInstances(Clock.class)) {
+            ((ClockMock)clock).fastForward(delta);
+        }
+    }
+
     protected void ensureWatcherStopped() throws Exception {
         ensureWatcherStopped(true);
     }