[Zen2] Update default for USE_ZEN2 to true (#35998)

Today the default for USE_ZEN2 is false and it is overridden in many places. By
defaulting it to true we can be sure that the only places in which Zen2 does
not work are those in which it is explicitly set to false.

server/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java

@@ -66,7 +66,7 @@ public class TransportClientIT extends ESIntegTestCase {
                 .put("transport.type", getTestTransportType())
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .put("cluster.name", "foobar")
-                .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
+                .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
                 .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1)
                 .build(), Arrays.asList(getTestTransportPlugin(), TestZenDiscovery.TestPlugin.class,
                                         MockHttpTransport.TestPlugin.class)).start()) {

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -1925,8 +1925,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
             initialNodeSettings.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
             initialTransportClientSettings.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
         }
-        if (addTestZenDiscovery()) {
+        if (addTestZenDiscovery() && getUseZen2() == false) {
-            initialNodeSettings.put(TestZenDiscovery.USE_ZEN2.getKey(), true);
+            initialNodeSettings.put(TestZenDiscovery.USE_ZEN2.getKey(), false);
         }
         return new NodeConfigurationSource() {
             @Override

test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java

@@ -191,7 +191,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("transport.type", getTestTransportType())
             .put(Node.NODE_DATA_SETTING.getKey(), true)
-            .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
+            .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
             .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
             // default the watermarks low values to prevent tests from failing on nodes without enough disk space
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -995,10 +995,21 @@ public abstract class ESTestCase extends LuceneTestCase {
     private static boolean useNio;
 
     @BeforeClass
-    public static void setUseNio() throws Exception {
+    public static void setUseNio() {
         useNio = randomBoolean();
     }
 
+    private static boolean useZen2;
+
+    @BeforeClass
+    public static void setUseZen2() {
+        useZen2 = true;
+    }
+
+    protected static boolean getUseZen2() {
+        return useZen2;
+    }
+
     public static String getTestTransportType() {
         return useNio ? MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME : MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME;
     }

test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java

@@ -64,7 +64,7 @@ public class TestZenDiscovery extends ZenDiscovery {
         Setting.boolSetting("discovery.zen.use_mock_pings", true, Setting.Property.NodeScope);
 
     public static final Setting<Boolean> USE_ZEN2 =
-        Setting.boolSetting("discovery.zen.use_zen2", false, Setting.Property.NodeScope);
+        Setting.boolSetting("discovery.zen.use_zen2", true, Setting.Property.NodeScope);
 
     public static final Setting<Boolean> USE_ZEN2_PERSISTED_STATE =
         Setting.boolSetting("discovery.zen.use_zen2_persisted_state", false, Setting.Property.NodeScope);

test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -27,7 +27,9 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.SettingsBasedHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.plugins.Plugin;
@@ -55,10 +57,12 @@ import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
 import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
 import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
+import static org.elasticsearch.test.discovery.TestZenDiscovery.USE_ZEN2;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.Matchers.equalTo;
@@ -190,11 +194,14 @@ public class InternalTestClusterTests extends ESTestCase {
                     .put(
                         NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(),
                         2 * ((masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes))
+                    .put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file")
+                    .putList(SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey())
                     .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
                 if (autoManageMinMasterNodes == false) {
                     assert minNumDataNodes == maxNumDataNodes;
                     assert masterNodes == false;
-                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1);
+                    settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1)
+                        .put(INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), minNumDataNodes);
                 }
                 return settings.build();
             }
@@ -265,6 +272,7 @@ public class InternalTestClusterTests extends ESTestCase {
                         NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(),
                         2 + (masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes)
                     .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType())
+                    .put(USE_ZEN2.getKey(), false) // full cluster restarts not yet supported
                     .build();
             }
 
@@ -375,6 +383,7 @@ public class InternalTestClusterTests extends ESTestCase {
                         // speedup join timeout as setting initial state timeout to 0 makes split
                         // elections more likely
                         .put(ZenDiscovery.JOIN_TIMEOUT_SETTING.getKey(), "3s")
+                        .put(USE_ZEN2.getKey(), false) // full cluster restarts not yet supported
                         .build();
             }
 
@@ -457,6 +466,7 @@ public class InternalTestClusterTests extends ESTestCase {
                 return Settings.builder()
                     .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
                     .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType())
+                    .put(USE_ZEN2.getKey(), false) // full cluster restarts not yet supported
                     .build();
             }
 

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java

@@ -174,6 +174,7 @@ public abstract class CcrIntegTestCase extends ESTestCase {
         builder.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         builder.putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes
         builder.putList(DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file");
+        builder.put(TestZenDiscovery.USE_ZEN2.getKey(), false); // some tests do full cluster restarts
         builder.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
         builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
         builder.put(XPackSettings.MONITORING_ENABLED.getKey(), false);

x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java

@@ -300,7 +300,7 @@ public class LicensingTests extends SecurityIntegTestCase {
             .put("path.home", home)
             .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false)
             .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen")
-            .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
+            .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
             .putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey())
             .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList)
             .build();

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java

@@ -33,6 +33,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.zen.SettingsBasedHostsProvider;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.plugins.MetaDataUpgrader;
 import org.elasticsearch.plugins.Plugin;
@@ -43,6 +45,7 @@ import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportInfo;
@@ -78,8 +81,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import static java.util.Collections.emptyMap;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.InternalTestCluster.clusterName;
 import static org.elasticsearch.xpack.security.audit.index.IndexNameResolver.Rollover.DAILY;
@@ -89,11 +92,11 @@ import static org.elasticsearch.xpack.security.audit.index.IndexNameResolver.Rol
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.hasSize;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -183,6 +186,9 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
             public Settings nodeSettings(int nodeOrdinal) {
                 Settings.Builder builder = Settings.builder()
                         .put(super.nodeSettings(nodeOrdinal))
+                        .put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file")
+                        .putList(SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey())
+                        .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
                         .put("xpack.security.audit.index.settings.index.number_of_shards", numShards)
                         .put("xpack.security.audit.index.settings.index.number_of_replicas", numReplicas)
                         // Disable native ML autodetect_process as the c++ controller won't be available

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java

@@ -108,7 +108,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
                 .put("path.home", home)
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
-                .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
+                .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
                 .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
                 //.put("xpack.ml.autodetect_process", false);
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(
@@ -152,7 +152,7 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 .put("discovery.initial_state_timeout", "0s")
                 .put("path.home", home)
                 .put(Node.NODE_MASTER_SETTING.getKey(), false)
-                .put(TestZenDiscovery.USE_ZEN2.getKey(), true)
+                .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2())
                 .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
                 //.put("xpack.ml.autodetect_process", false);
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(