diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 85b61e2c986..13b983c8a5e 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -1116,7 +1116,6 @@ - @@ -1137,7 +1136,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 8bbd6f09d7e..3101a2c04cc 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -20,15 +20,12 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.misc.IndexMergeTool; import org.elasticsearch.Version; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.MapperService; @@ -36,10 +33,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; import java.util.Collections; -import java.util.Map; -import java.util.Set; -import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.common.util.set.Sets.newHashSet; /** @@ -53,13 +47,13 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; public class MetaDataIndexUpgradeService extends AbstractComponent { private final MapperRegistry mapperRegistry; - private final IndexScopedSettings indexScopedSettigns; + private final IndexScopedSettings indexScopedSettings; @Inject public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings) { super(settings); this.mapperRegistry = mapperRegistry; - this.indexScopedSettigns = indexScopedSettings; + this.indexScopedSettings = indexScopedSettings; } /** @@ -182,39 +176,13 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { } } - private static final String ARCHIVED_SETTINGS_PREFIX = "archived."; - IndexMetaData archiveBrokenIndexSettings(IndexMetaData indexMetaData) { - Settings settings = indexMetaData.getSettings(); - Settings.Builder builder = Settings.builder(); - boolean changed = false; - for (Map.Entry entry : settings.getAsMap().entrySet()) { - try { - Setting setting = indexScopedSettigns.get(entry.getKey()); - if (setting != null) { - setting.get(settings); - builder.put(entry.getKey(), entry.getValue()); - } else { - if (indexScopedSettigns.isPrivateSetting(entry.getKey()) || entry.getKey().startsWith(ARCHIVED_SETTINGS_PREFIX)) { - builder.put(entry.getKey(), entry.getValue()); - } else { - changed = true; - logger.warn("[{}] found unknown index setting: {} value: {} - archiving", indexMetaData.getIndex(), entry.getKey(), entry.getValue()); - // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there - // but we want users to be aware that some of their setting are broken and they can research why and what they need to do to replace them. - builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue()); - } - } - } catch (IllegalArgumentException ex) { - changed = true; - logger.warn("[{}] found invalid index setting: {} value: {} - archiving",ex, indexMetaData.getIndex(), entry.getKey(), entry.getValue()); - // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there - // but we want users to be aware that some of their setting sare broken and they can research why and what they need to do to replace them. - builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue()); - } + final Settings settings = indexMetaData.getSettings(); + final Settings upgrade = indexScopedSettings.archiveUnknownOrBrokenSettings(settings); + if (upgrade != settings) { + return IndexMetaData.builder(indexMetaData).settings(upgrade).build(); + } else { + return indexMetaData; } - - return changed ? IndexMetaData.builder(indexMetaData).settings(builder.build()).build() : indexMetaData; } - } diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java index fa9b3492685..54e8535b57e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java +++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java @@ -1000,4 +1000,8 @@ public class ClusterService extends AbstractLifecycleComponent { } } } + + public ClusterSettings getClusterSettings() { + return clusterSettings; + } } diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java index 410adc82da1..358706c9d3f 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java @@ -48,6 +48,7 @@ import java.util.stream.Collectors; * This service offers transactional application of updates settings. */ public abstract class AbstractScopedSettings extends AbstractComponent { + public static final String ARCHIVED_SETTINGS_PREFIX = "archived."; private Settings lastSettingsApplied = Settings.EMPTY; private final List> settingUpdaters = new CopyOnWriteArrayList<>(); private final Map> complexMatchers; @@ -478,4 +479,53 @@ public abstract class AbstractScopedSettings extends AbstractComponent { } return null; } + + /** + * Archives broken or unknown settings. Any setting that is not recognized or fails + * validation will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX} + * and remains in the settings object. This can be used to detect broken settings via APIs. + */ + public Settings archiveUnknownOrBrokenSettings(Settings settings) { + Settings.Builder builder = Settings.builder(); + boolean changed = false; + for (Map.Entry entry : settings.getAsMap().entrySet()) { + try { + Setting setting = get(entry.getKey()); + if (setting != null) { + setting.get(settings); + builder.put(entry.getKey(), entry.getValue()); + } else { + if (entry.getKey().startsWith(ARCHIVED_SETTINGS_PREFIX) || isPrivateSetting(entry.getKey())) { + builder.put(entry.getKey(), entry.getValue()); + } else { + changed = true; + logger.warn("found unknown setting: {} value: {} - archiving", entry.getKey(), entry.getValue()); + // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there + // but we want users to be aware that some of their setting are broken and they can research why and what they need to do to replace them. + builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue()); + } + } + } catch (IllegalArgumentException ex) { + changed = true; + logger.warn("found invalid setting: {} value: {} - archiving",ex , entry.getKey(), entry.getValue()); + // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there + // but we want users to be aware that some of their setting sare broken and they can research why and what they need to do to replace them. + builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue()); + } + } + if (changed) { + return builder.build(); + } else { + return settings; + } + } + + /** + * Returns true iff the setting is a private setting ie. it should be treated as valid even though it has no internal + * representation. Otherwise false + */ + // TODO this should be replaced by Setting.Property.HIDDEN or something like this. + protected boolean isPrivateSetting(String key) { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 322ac4de799..3fa9cdcddb0 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -171,7 +171,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { super.validateSettingKey(setting); } - public boolean isPrivateSetting(String key) { + @Override + protected final boolean isPrivateSetting(String key) { switch (key) { case IndexMetaData.SETTING_CREATION_DATE: case IndexMetaData.SETTING_INDEX_UUID: diff --git a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java index 3640d3e4bec..221dc234511 100644 --- a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java +++ b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java @@ -24,12 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.gateway.MetaDataStateFormat; -import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java index 54c270e2072..b2cb2d11079 100644 --- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -20,7 +20,6 @@ package org.elasticsearch.gateway; import com.carrotsearch.hppc.ObjectFloatHashMap; -import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.FailedNodeException; @@ -31,18 +30,16 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.indices.IndicesService; -import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collections; import java.util.function.Supplier; /** @@ -155,6 +152,9 @@ public class Gateway extends AbstractComponent implements ClusterStateListener { } } } + final ClusterSettings clusterSettings = clusterService.getClusterSettings(); + metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.persistentSettings())); + metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.transientSettings())); ClusterState.Builder builder = ClusterState.builder(clusterService.state().getClusterName()); builder.metaData(metaDataBuilder); listener.onSuccess(builder.build()); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index af2894833cb..55955ba269d 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Priority; import org.elasticsearch.common.logging.ESLogger; @@ -35,6 +36,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperParsingException; @@ -46,6 +48,8 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; +import java.io.IOException; + import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -65,14 +69,16 @@ public class GatewayIndexStateIT extends ESIntegTestCase { logger.info("--> creating test index, with meta routing"); client().admin().indices().prepareCreate("test") - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) + .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing") + .field("required", true).endObject().endObject().endObject()) .execute().actionGet(); logger.info("--> waiting for yellow status"); ensureYellow(); logger.info("--> verify meta _routing required exists"); - MappingMetaData mappingMd = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test").mapping("type1"); + MappingMetaData mappingMd = client().admin().cluster().prepareState().execute().actionGet().getState().metaData() + .index("test").mapping("type1"); assertThat(mappingMd.routing().required(), equalTo(true)); logger.info("--> restarting nodes..."); @@ -101,7 +107,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ClusterStateResponse stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); - assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); + assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), + equalTo(test.totalNumShards)); logger.info("--> indexing a simple document"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); @@ -138,7 +145,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); - assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); + assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), + equalTo(test.totalNumShards)); logger.info("--> trying to get the indexed document on the first index"); GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); @@ -176,7 +184,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN)); assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries)); - assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); + assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), + equalTo(test.totalNumShards)); logger.info("--> trying to get the indexed document on the first round (before close and shutdown)"); getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); @@ -202,7 +211,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).build()); logger.info("--> waiting for test index to be created"); - ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setIndices("test").execute().actionGet(); + ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setIndices("test") + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); logger.info("--> verify we have an index"); @@ -236,7 +246,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet(); logger.info("--> waiting for green status"); - ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet(); + ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus() + .setWaitForNodes("2").execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); logger.info("--> verify 1 doc in the index"); @@ -255,7 +266,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { client().admin().indices().prepareOpen("test").execute().actionGet(); logger.info("--> waiting for green status"); - health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet(); + health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2") + .execute().actionGet(); assertThat(health.isTimedOut(), equalTo(false)); logger.info("--> verify 1 doc in the index"); @@ -300,7 +312,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase { ensureGreen(); // make sure that any other events were processed - assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get().isTimedOut()); + assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get() + .isTimedOut()); logger.info("--> verify we read the right thing through alias"); assertThat(client().prepareGet("test", "type1", "2").execute().actionGet().isExists(), equalTo(true)); @@ -492,4 +505,44 @@ public class GatewayIndexStateIT extends ESIntegTestCase { logger.info("--> verify 1 doc in the index"); assertHitCount(client().prepareSearch().setQuery(matchQuery("field1", "value one")).get(), 1L); } + + public void testArchiveBrokenClusterSettings() throws Exception { + logger.info("--> starting one node"); + internalCluster().startNode(); + client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet(); + logger.info("--> waiting for green status"); + if (usually()) { + ensureYellow(); + } else { + internalCluster().startNode(); + client().admin().cluster() + .health(Requests.clusterHealthRequest() + .waitForGreenStatus() + .waitForEvents(Priority.LANGUID) + .waitForRelocatingShards(0).waitForNodes("2")).actionGet(); + } + ClusterState state = client().admin().cluster().prepareState().get().getState(); + MetaData metaData = state.getMetaData(); + for (NodeEnvironment nodeEnv : internalCluster().getInstances(NodeEnvironment.class)) { + MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() + .put(metaData.persistentSettings()).put("this.is.unknown", true) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); + MetaData.FORMAT.write(brokenMeta, metaData.version(), nodeEnv.nodeDataPaths()); + } + internalCluster().fullRestart(); + ensureYellow("test"); // wait for state recovery + state = client().admin().cluster().prepareState().get().getState(); + assertEquals("true", state.metaData().persistentSettings().get("archived.this.is.unknown")); + assertEquals("broken", state.metaData().persistentSettings().get("archived." + + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())); + + // delete these settings + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().putNull("archived.*")).get(); + + state = client().admin().cluster().prepareState().get().getState(); + assertNull(state.metaData().persistentSettings().get("archived.this.is.unknown")); + assertNull(state.metaData().persistentSettings().get("archived." + + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())); + assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); + } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 46d99e3b4bc..9d74b1ce429 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -28,10 +29,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -40,10 +43,10 @@ import java.util.function.Function; public class IndexSettingsTests extends ESTestCase { - public void testRunListener() { Version version = VersionUtils.getPreviousVersion(); - Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build(); + Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version) + .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build(); final AtomicInteger integer = new AtomicInteger(0); Setting integerSetting = Setting.intSetting("index.test.setting.int", -1, Property.Dynamic, Property.IndexScope); @@ -57,7 +60,8 @@ public class IndexSettingsTests extends ESTestCase { assertFalse(settings.updateIndexMetaData(metaData)); assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap()); assertEquals(0, integer.get()); - assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42).build()))); + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42) + .build()))); assertEquals(42, integer.get()); } @@ -77,13 +81,15 @@ public class IndexSettingsTests extends ESTestCase { settings.getScopedSettings().addSettingsUpdateConsumer(notUpdated, builder::append); assertEquals(0, integer.get()); assertEquals("", builder.toString()); - IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.test.setting.int", 42).build()); + IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()) + .put("index.test.setting.int", 42).build()); assertTrue(settings.updateIndexMetaData(newMetaData)); assertSame(settings.getIndexMetaData(), newMetaData); assertEquals(42, integer.get()); assertEquals("", builder.toString()); integer.set(0); - assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.not.updated", "boom").build()))); + assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()) + .put("index.not.updated", "boom").build()))); assertEquals("boom", builder.toString()); assertEquals("not updated - we preserve the old settings", 0, integer.get()); @@ -91,21 +97,25 @@ public class IndexSettingsTests extends ESTestCase { public void testSettingsConsistency() { Version version = VersionUtils.getPreviousVersion(); - IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()); + IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version) + .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(version, settings.getIndexVersionCreated()); assertEquals("_na_", settings.getUUID()); try { - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, + Version.CURRENT).put("index.test.setting.int", 42).build())); fail("version has changed"); } catch (IllegalArgumentException ex) { assertTrue(ex.getMessage(), ex.getMessage().startsWith("version mismatch on settings update expected: ")); } - metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build()); + metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build()); settings = new IndexSettings(metaData, Settings.EMPTY); try { - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, + Version.CURRENT).put("index.test.setting.int", 42).build())); fail("uuid missing/change"); } catch (IllegalArgumentException ex) { assertEquals("uuid mismatch on settings update expected: 0xdeadbeef but was: _na_", ex.getMessage()); @@ -118,7 +128,8 @@ public class IndexSettingsTests extends ESTestCase { if (settings.length > 0) { settingSet.addAll(Arrays.asList(settings)); } - return new IndexSettings(metaData, nodeSettings, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), new IndexScopedSettings(Settings.EMPTY, settingSet)); + return new IndexSettings(metaData, nodeSettings, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), + new IndexScopedSettings(Settings.EMPTY, settingSet)); } @@ -172,7 +183,8 @@ public class IndexSettingsTests extends ESTestCase { .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(Translog.Durability.ASYNC, settings.getTranslogDurability()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "request").build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), + "request").build())); assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability()); metaData = newIndexMeta("index", Settings.settingsBuilder() @@ -189,7 +201,8 @@ public class IndexSettingsTests extends ESTestCase { .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertFalse(settings.isWarmerEnabled()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_WARMER_ENABLED_SETTING.getKey(), "true").build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_WARMER_ENABLED_SETTING.getKey(), + "true").build())); assertTrue(settings.isWarmerEnabled()); metaData = newIndexMeta("index", Settings.settingsBuilder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -205,10 +218,13 @@ public class IndexSettingsTests extends ESTestCase { .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval) .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); - assertEquals(TimeValue.parseTimeValue(refreshInterval, new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval()); + assertEquals(TimeValue.parseTimeValue(refreshInterval, new TimeValue(1, TimeUnit.DAYS), + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval()); String newRefreshInterval = getRandomTimeString(); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), newRefreshInterval).build())); - assertEquals(TimeValue.parseTimeValue(newRefreshInterval, new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval()); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), + newRefreshInterval).build())); + assertEquals(TimeValue.parseTimeValue(newRefreshInterval, new TimeValue(1, TimeUnit.DAYS), + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval()); } private String getRandomTimeString() { @@ -227,7 +243,8 @@ public class IndexSettingsTests extends ESTestCase { .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(15, settings.getMaxResultWindow()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), 42).build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), + 42).build())); assertEquals(42, settings.getMaxResultWindow()); settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY)); assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow()); @@ -246,11 +263,15 @@ public class IndexSettingsTests extends ESTestCase { .put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), gcDeleteSetting.getStringRep()) .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); - assertEquals(TimeValue.parseTimeValue(gcDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis()); + assertEquals(TimeValue.parseTimeValue(gcDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), + IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis()); TimeValue newGCDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), newGCDeleteSetting.getStringRep()).build())); - assertEquals(TimeValue.parseTimeValue(newGCDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), randomBoolean() ? -1 : new TimeValue(-1, TimeUnit.MILLISECONDS)).build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), + newGCDeleteSetting.getStringRep()).build())); + assertEquals(TimeValue.parseTimeValue(newGCDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), + IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis()); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), + randomBoolean() ? -1 : new TimeValue(-1, TimeUnit.MILLISECONDS)).build())); assertEquals(-1, settings.getGcDeletesInMillis()); } @@ -261,7 +282,8 @@ public class IndexSettingsTests extends ESTestCase { .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertFalse(settings.isTTLPurgeDisabled()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TTL_DISABLE_PURGE_SETTING.getKey(), "true").build())); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TTL_DISABLE_PURGE_SETTING.getKey(), + "true").build())); assertTrue(settings.isTTLPurgeDisabled()); settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY)); @@ -276,7 +298,8 @@ public class IndexSettingsTests extends ESTestCase { public void testTranslogFlushSizeThreshold() { ByteSizeValue translogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt())); - ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); + ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(), + IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.toString()) @@ -284,8 +307,33 @@ public class IndexSettingsTests extends ESTestCase { IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(actualValue, settings.getFlushThresholdSize()); ByteSizeValue newTranslogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt())); - ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build())); + ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(), + IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build())); assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize()); } + + + public void testArchiveBrokenIndexSettings() { + Settings settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.EMPTY); + assertSame(settings, Settings.EMPTY); + settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder() + .put("index.refresh_interval", "-200").build()); + assertEquals("-200", settings.get("archived.index.refresh_interval")); + assertNull(settings.get("index.refresh_interval")); + + Settings prevSettings = settings; // no double archive + settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(prevSettings); + assertSame(prevSettings, settings); + + settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder() + .put("index.version.created", Version.CURRENT.id) // private setting + .put("index.unknown", "foo") + .put("index.refresh_interval", "2s").build()); + + assertEquals("foo", settings.get("archived.index.unknown")); + assertEquals(Integer.toString(Version.CURRENT.id), settings.get("index.version.created")); + assertEquals("2s", settings.get("index.refresh_interval")); + } }