From 4e72f3c5c63f1c8710e56c549a62b27223c1ea4a Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 16 Jan 2019 11:14:16 -0500 Subject: [PATCH] DedicatedClusterSnapshotRestoreIT to Zen2 (#37489) All tests except testRestorePersistentSettings (renamed to testExceptionWhenRestoringPersistentSettings) worked fine. testExceptionWhenRestoringPersistentSettings re-written to use a custom setting, because "minimum master node" setting is no longer available in Zen2. It turns out there is no good replacement for "minimum master node" setting for this test, that's why the custom setting is introduced. Unfortunately, there is #37485 bug and currently RestoreService does not perform setting validation. That's why the test is annotated with @AwaitsFix, the idea is to merge this commit and then fix the issue and enable the test. (The test passes with a simple fix, that adds a single line to RestoreService). --- .../DedicatedClusterSnapshotRestoreIT.java | 94 ++++++++++--------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index afdb14eaf6b..554231982da 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -53,6 +53,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -60,7 +61,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.node.Node; @@ -78,7 +78,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.TestCustomMetaData; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.disruption.BusyMasterServiceDisruption; import org.elasticsearch.test.disruption.ServiceDisruptionScheme; import org.elasticsearch.test.rest.FakeRestRequest; @@ -93,11 +92,13 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; @@ -118,13 +119,6 @@ import static org.mockito.Mockito.mock; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0) public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // requires more work - .build(); - } - public static class TestCustomMetaDataPlugin extends Plugin { private final List namedWritables = new ArrayList<>(); @@ -168,26 +162,53 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest @Override protected Collection> nodePlugins() { - return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class); + return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class, BrokenSettingPlugin.class); } - public void testRestorePersistentSettings() throws Exception { + public static class BrokenSettingPlugin extends Plugin { + private static boolean breakSetting = false; + private static final IllegalArgumentException EXCEPTION = new IllegalArgumentException("this setting goes boom"); + + static void breakSetting(boolean breakSetting) { + BrokenSettingPlugin.breakSetting = breakSetting; + } + + static final Setting BROKEN_SETTING = new Setting<>("setting.broken", "default", s->s, + s-> { + if ((s.equals("default") == false && breakSetting)) { + throw EXCEPTION; + } + }, + Setting.Property.NodeScope, Setting.Property.Dynamic); + + @Override + public List> getSettings() { + return Collections.singletonList(BROKEN_SETTING); + } + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37485") + public void testExceptionWhenRestoringPersistentSettings() { logger.info("--> start 2 nodes"); - internalCluster().startNode(); + internalCluster().startNodes(2); + Client client = client(); - String secondNode = internalCluster().startNode(); - logger.info("--> wait for the second node to join the cluster"); - assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut(), equalTo(false)); + Consumer setSettingValue = value -> { + client.admin().cluster().prepareUpdateSettings().setPersistentSettings( + Settings.builder() + .put(BrokenSettingPlugin.BROKEN_SETTING.getKey(), value)) + .execute().actionGet(); + }; + + Consumer assertSettingValue = value -> { + assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState() + .getMetaData().persistentSettings().get(BrokenSettingPlugin.BROKEN_SETTING.getKey()), + equalTo(value)); + }; logger.info("--> set test persistent setting"); - client.admin().cluster().prepareUpdateSettings().setPersistentSettings( - Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)) - .execute().actionGet(); - - assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState() - .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), - equalTo(2)); + setSettingValue.accept("new value"); + assertSettingValue.accept("new value"); logger.info("--> create repository"); AcknowledgedResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") @@ -203,32 +224,21 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest .getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - logger.info("--> clean the test persistent setting"); - client.admin().cluster().prepareUpdateSettings().setPersistentSettings( - Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1)) - .execute().actionGet(); - assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState() - .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), - equalTo(1)); - - stopNode(secondNode); - assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("1").get().isTimedOut(), equalTo(false)); + logger.info("--> change the test persistent setting and break it"); + setSettingValue.accept("new value 2"); + assertSettingValue.accept("new value 2"); + BrokenSettingPlugin.breakSetting(true); logger.info("--> restore snapshot"); try { client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true) .setWaitForCompletion(true).execute().actionGet(); - fail("can't restore minimum master nodes"); + } catch (IllegalArgumentException ex) { - assertEquals("illegal value can't update [discovery.zen.minimum_master_nodes] from [1] to [2]", ex.getMessage()); - assertEquals("cannot set discovery.zen.minimum_master_nodes to more than the current master nodes count [1]", - ex.getCause().getMessage()); + assertEquals(BrokenSettingPlugin.EXCEPTION.getMessage(), ex.getMessage()); } - logger.info("--> ensure that zen discovery minimum master nodes wasn't restored"); - assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState() - .getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), - not(equalTo(2))); + + assertSettingValue.accept("new value 2"); } public void testRestoreCustomMetadata() throws Exception {