DedicatedClusterSnapshotRestoreIT to Zen2 (#37489)

All tests except testRestorePersistentSettings (renamed to
testExceptionWhenRestoringPersistentSettings) worked fine.
testExceptionWhenRestoringPersistentSettings re-written to use a custom
setting, because "minimum master node" setting is no longer available
in Zen2. It turns out there is no good replacement for "minimum master
node" setting for this test, that's why the custom setting is
introduced.

Unfortunately, there is #37485 bug and currently
RestoreService does not perform setting validation. That's why the
test is annotated with @AwaitsFix, the idea is to merge this commit and
then fix the issue and enable the test. (The test passes with a simple
fix, that adds a single line to RestoreService).

server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -53,6 +53,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -60,7 +61,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.node.Node;
@@ -78,7 +78,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.TestCustomMetaData;
-import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.disruption.BusyMasterServiceDisruption;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -93,11 +92,13 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
@@ -118,13 +119,6 @@ import static org.mockito.Mockito.mock;
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
 public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-            .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // requires more work
-            .build();
-    }
-
     public static class TestCustomMetaDataPlugin extends Plugin {
 
         private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
@@ -168,26 +162,53 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class);
+        return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class, BrokenSettingPlugin.class);
     }
 
-    public void testRestorePersistentSettings() throws Exception {
+    public static class BrokenSettingPlugin extends Plugin {
-        logger.info("--> start 2 nodes");
+        private static boolean breakSetting = false;
-        internalCluster().startNode();
+        private static final IllegalArgumentException EXCEPTION =  new IllegalArgumentException("this setting goes boom");
-        Client client = client();
-        String secondNode = internalCluster().startNode();
-        logger.info("--> wait for the second node to join the cluster");
-        assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut(), equalTo(false));
 
-        logger.info("--> set test persistent setting");
+        static void breakSetting(boolean breakSetting) {
+            BrokenSettingPlugin.breakSetting = breakSetting;
+        }
+
+        static final Setting<String> BROKEN_SETTING = new Setting<>("setting.broken", "default", s->s,
+                s-> {
+                    if ((s.equals("default") == false && breakSetting)) {
+                        throw EXCEPTION;
+                    }
+                },
+                Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(BROKEN_SETTING);
+        }
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37485")
+    public void testExceptionWhenRestoringPersistentSettings() {
+        logger.info("--> start 2 nodes");
+        internalCluster().startNodes(2);
+
+        Client client = client();
+        Consumer<String> setSettingValue = value -> {
             client.admin().cluster().prepareUpdateSettings().setPersistentSettings(
                     Settings.builder()
-                        .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2))
+                            .put(BrokenSettingPlugin.BROKEN_SETTING.getKey(), value))
                     .execute().actionGet();
+        };
 
+        Consumer<String> assertSettingValue = value -> {
             assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState()
-                .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1),
+                            .getMetaData().persistentSettings().get(BrokenSettingPlugin.BROKEN_SETTING.getKey()),
-            equalTo(2));
+                    equalTo(value));
+        };
+
+        logger.info("--> set test persistent setting");
+        setSettingValue.accept("new value");
+        assertSettingValue.accept("new value");
 
         logger.info("--> create repository");
         AcknowledgedResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
@@ -203,32 +224,21 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
                 .getSnapshots().get(0).state(),
             equalTo(SnapshotState.SUCCESS));
 
-        logger.info("--> clean the test persistent setting");
+        logger.info("--> change the test persistent setting and break it");
-        client.admin().cluster().prepareUpdateSettings().setPersistentSettings(
+        setSettingValue.accept("new value 2");
-                Settings.builder()
+        assertSettingValue.accept("new value 2");
-                        .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1))
+        BrokenSettingPlugin.breakSetting(true);
-                .execute().actionGet();
-        assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState()
-                .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1),
-            equalTo(1));
-
-        stopNode(secondNode);
-        assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("1").get().isTimedOut(), equalTo(false));
 
         logger.info("--> restore snapshot");
         try {
             client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true)
                 .setWaitForCompletion(true).execute().actionGet();
-            fail("can't restore minimum master nodes");
+
         } catch (IllegalArgumentException ex) {
-            assertEquals("illegal value can't update [discovery.zen.minimum_master_nodes] from [1] to [2]", ex.getMessage());
+            assertEquals(BrokenSettingPlugin.EXCEPTION.getMessage(), ex.getMessage());
-            assertEquals("cannot set discovery.zen.minimum_master_nodes to more than the current master nodes count [1]",
-                ex.getCause().getMessage());
         }
-        logger.info("--> ensure that zen discovery minimum master nodes wasn't restored");
+
-        assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState()
+        assertSettingValue.accept("new value 2");
-                .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1),
-            not(equalTo(2)));
     }
 
     public void testRestoreCustomMetadata() throws Exception {