From c1685c70ea1372ffbe39f4be980c264f65d89f14 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 19 Jun 2017 16:12:08 +0200 Subject: [PATCH] Revert "X-Pack: Add index.internal.format index setting (elastic/x-pack-elasticsearch#1700)" This reverts commit elastic/x-pack-elasticsearch@e430691c511368c047ccd06b9653691c4f548f24. The setting is going to be applied in core, so that only parts of these changes need to be applied here (in a seperate PR). Original commit: elastic/x-pack-elasticsearch@9e178c87faab69cbb080a5a792cc227ebfb8dad0 --- .../org/elasticsearch/xpack/XPackPlugin.java | 14 ++--- .../watcher/WatcherLifeCycleService.java | 34 +--------- .../src/main/resources/monitoring-alerts.json | 1 - .../src/main/resources/monitoring-beats.json | 1 - plugin/src/main/resources/monitoring-es.json | 3 +- .../src/main/resources/monitoring-kibana.json | 3 +- .../main/resources/monitoring-logstash.json | 3 +- .../resources/security-index-template.json | 1 - .../main/resources/security_audit_log.json | 3 +- .../src/main/resources/triggered-watches.json | 3 +- plugin/src/main/resources/watch-history.json | 1 - plugin/src/main/resources/watches.json | 3 +- .../watcher/WatcherLifeCycleServiceTests.java | 62 ------------------- 13 files changed, 16 insertions(+), 116 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java index 5dbf245cd7e..2c73cb582b9 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java @@ -46,6 +46,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -106,7 +107,11 @@ import java.io.IOException; import java.nio.file.Path; import java.security.AccessController; import java.security.GeneralSecurityException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; import java.security.PrivilegedAction; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; @@ -146,11 +151,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I // inside of YAML settings we still use xpack do not having handle issues with dashes private static final String SETTINGS_NAME = "xpack"; - // internal index format is used for upgrading - public static final Setting INDEX_INTERNAL_FORMAT_SETTING = - Setting.intSetting("index.internal.format", 0, Setting.Property.IndexScope); - public static final int INTERNAL_INDEX_FORMAT = 6; - // TODO: clean up this library to not ask for write access to all system properties! static { // invoke this clinit in unbound with permissions to access all system properties @@ -359,8 +359,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I settings.addAll(licensing.getSettings()); settings.addAll(XPackSettings.getAllSettings()); - // an internal index format description, allowing us to find out if this index is upgraded or needs upgrading - settings.add(INDEX_INTERNAL_FORMAT_SETTING); + // we add the `xpack.version` setting to all internal indices + settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope)); // notification services settings.add(SlackService.SLACK_ACCOUNT_SETTING); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java index 6a213131c88..a142cfe4984 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java @@ -9,7 +9,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.RoutingNode; @@ -20,8 +19,6 @@ import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.XPackPlugin; -import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; import org.elasticsearch.xpack.watcher.watch.Watch; import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils; @@ -95,10 +92,8 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste } } - /** - * - * @param event The event of containing the new cluster state - * + /* + * TODO possible optimization * stop certain parts of watcher, when there are no watcher indices on this node by checking the shardrouting * note that this is not easily possible, because of the execute watch api, that needs to be able to execute anywhere! * this means, only certain components can be stopped @@ -126,7 +121,6 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste DiscoveryNode localNode = event.state().nodes().getLocalNode(); RoutingNode routingNode = event.state().getRoutingNodes().node(localNode.getId()); IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(Watch.INDEX, event.state().metaData()); - // no watcher index, time to pause, if we currently have shards here if (watcherIndexMetaData == null) { if (previousAllocationIds.get().isEmpty() == false) { @@ -157,33 +151,11 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste executor.execute(() -> watcherService.reload(event.state(), "different shard allocation ids")); } } else if (watcherService.state() != WatcherState.STARTED && watcherService.state() != WatcherState.STARTING) { - if (isIndexInternalFormat(event.state().metaData(), Watch.INDEX) && - isIndexInternalFormat(event.state().metaData(), TriggeredWatchStore.INDEX_NAME)) { - executor.execute(() -> start(event.state(), false)); - } else { - logger.error("Not starting watcher, the indices have not been upgraded yet. Please run the Upgrade API"); - } + executor.execute(() -> start(event.state(), false)); } } } - /** - * If the supplied index exists, ensure that the 'index.internal.format' setting is configured appropriately. - * This ensures that watcher can use this index as needed. - * - * @param metaData The cluster state meta data - * @param index The index name to check for. Aliases will be resolved properly. - * @return true if the index does not exist or the internal format is set properly - */ - boolean isIndexInternalFormat(MetaData metaData, String index) { - IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(index, metaData); - if (watcherIndexMetaData == null) { - return true; - } else { - return XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.get(watcherIndexMetaData.getSettings()) == XPackPlugin.INTERNAL_INDEX_FORMAT; - } - } - public WatcherMetaData watcherMetaData() { return watcherMetaData; } diff --git a/plugin/src/main/resources/monitoring-alerts.json b/plugin/src/main/resources/monitoring-alerts.json index bd153afdc52..8d05d89d079 100644 --- a/plugin/src/main/resources/monitoring-alerts.json +++ b/plugin/src/main/resources/monitoring-alerts.json @@ -5,7 +5,6 @@ "index": { "number_of_shards": 1, "number_of_replicas": 1, - "internal.format": 6, "codec": "best_compression" } }, diff --git a/plugin/src/main/resources/monitoring-beats.json b/plugin/src/main/resources/monitoring-beats.json index 0c771920ad0..a8c9410e8a1 100644 --- a/plugin/src/main/resources/monitoring-beats.json +++ b/plugin/src/main/resources/monitoring-beats.json @@ -4,7 +4,6 @@ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 1, - "index.internal.format": 6, "index.codec": "best_compression" }, "mappings": { diff --git a/plugin/src/main/resources/monitoring-es.json b/plugin/src/main/resources/monitoring-es.json index 6ac7de66f04..d91ba561a9c 100644 --- a/plugin/src/main/resources/monitoring-es.json +++ b/plugin/src/main/resources/monitoring-es.json @@ -4,8 +4,7 @@ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 1, - "index.codec": "best_compression", - "index.internal.format": 6 + "index.codec": "best_compression" }, "mappings": { "doc": { diff --git a/plugin/src/main/resources/monitoring-kibana.json b/plugin/src/main/resources/monitoring-kibana.json index 22d5b5ba6a3..72e64e15ef9 100644 --- a/plugin/src/main/resources/monitoring-kibana.json +++ b/plugin/src/main/resources/monitoring-kibana.json @@ -4,8 +4,7 @@ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 1, - "index.codec": "best_compression", - "index.internal.format": 6 + "index.codec": "best_compression" }, "mappings": { "doc": { diff --git a/plugin/src/main/resources/monitoring-logstash.json b/plugin/src/main/resources/monitoring-logstash.json index ec19cf07cde..c263c935503 100644 --- a/plugin/src/main/resources/monitoring-logstash.json +++ b/plugin/src/main/resources/monitoring-logstash.json @@ -4,8 +4,7 @@ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 1, - "index.codec": "best_compression", - "index.internal.format": 6 + "index.codec": "best_compression" }, "mappings": { "doc": { diff --git a/plugin/src/main/resources/security-index-template.json b/plugin/src/main/resources/security-index-template.json index 64e84f60541..6acac33b688 100644 --- a/plugin/src/main/resources/security-index-template.json +++ b/plugin/src/main/resources/security-index-template.json @@ -7,7 +7,6 @@ "number_of_replicas" : 0, "auto_expand_replicas" : "0-all", "index.priority": 1000, - "index.internal.format": 6, "analysis" : { "filter" : { "email" : { diff --git a/plugin/src/main/resources/security_audit_log.json b/plugin/src/main/resources/security_audit_log.json index 1753d06265e..2614e4e32e2 100644 --- a/plugin/src/main/resources/security_audit_log.json +++ b/plugin/src/main/resources/security_audit_log.json @@ -2,8 +2,7 @@ "index_patterns": ".security_audit_log*", "order": 2147483647, "settings": { - "index.mapper.dynamic" : false, - "index.internal.format": 6 + "index.mapper.dynamic" : false }, "mappings": { "event": { diff --git a/plugin/src/main/resources/triggered-watches.json b/plugin/src/main/resources/triggered-watches.json index 294ad47928b..c8b5db73060 100644 --- a/plugin/src/main/resources/triggered-watches.json +++ b/plugin/src/main/resources/triggered-watches.json @@ -5,8 +5,7 @@ "index.number_of_shards": 1, "index.mapper.dynamic" : false, "index.refresh_interval" : "-1", - "index.priority": 900, - "index.internal.format": 6 + "index.priority": 900 }, "mappings": { "doc": { diff --git a/plugin/src/main/resources/watch-history.json b/plugin/src/main/resources/watch-history.json index 85adabdf6e4..57fbee0c463 100644 --- a/plugin/src/main/resources/watch-history.json +++ b/plugin/src/main/resources/watch-history.json @@ -3,7 +3,6 @@ "order": 2147483647, "settings": { "xpack.watcher.template.version": "${xpack.watcher.template.version}", - "index.internal.format": 6, "index.number_of_shards": 1, "index.mapper.dynamic": false }, diff --git a/plugin/src/main/resources/watches.json b/plugin/src/main/resources/watches.json index 7e4607668fa..fe26b1a1988 100644 --- a/plugin/src/main/resources/watches.json +++ b/plugin/src/main/resources/watches.json @@ -4,8 +4,7 @@ "settings": { "index.number_of_shards": 1, "index.mapper.dynamic" : false, - "index.priority": 800, - "index.internal.format": 6 + "index.priority": 800 }, "mappings": { "doc": { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java index 670d62c4d49..809731071d8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java @@ -28,8 +28,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.XPackPlugin; -import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore; import org.elasticsearch.xpack.watcher.watch.Watch; import org.junit.Before; import org.mockito.stubbing.Answer; @@ -41,7 +39,6 @@ import java.util.concurrent.ExecutorService; import static java.util.Arrays.asList; import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; @@ -355,65 +352,6 @@ public class WatcherLifeCycleServiceTests extends ESTestCase { verify(watcherService, times(1)).pauseExecution(anyObject()); } - public void testWatcherDoesNotStartWithOldIndexFormat() throws Exception { - String index = randomFrom(Watch.INDEX, TriggeredWatchStore.INDEX_NAME); - Index watchIndex = new Index(index, "foo"); - ShardId shardId = new ShardId(watchIndex, 0); - IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(watchIndex) - .addShard(TestShardRouting.newShardRouting(shardId, "node_1", true, STARTED)).build(); - DiscoveryNodes nodes = new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")).build(); - - Settings.Builder indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); - if (randomBoolean()) { - int randomNumber = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1), - randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT, Integer.MAX_VALUE)); - indexSettings.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), randomNumber); - } - IndexMetaData.Builder newIndexMetaDataBuilder = IndexMetaData.builder(index).settings(indexSettings); - - ClusterState clusterStateWithWatcherIndex = ClusterState.builder(new ClusterName("my-cluster")) - .nodes(nodes) - .routingTable(RoutingTable.builder().add(watchRoutingTable).build()) - .metaData(MetaData.builder().put(newIndexMetaDataBuilder)) - .build(); - - ClusterState emptyClusterState = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build(); - - when(watcherService.state()).thenReturn(WatcherState.STOPPED); - when(watcherService.validate(eq(clusterStateWithWatcherIndex))).thenReturn(true); - lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterStateWithWatcherIndex, emptyClusterState)); - verify(watcherService, never()).start(any(ClusterState.class)); - } - - public void testInternalIndexFormatCheck() { - int indexFormat = XPackPlugin.INTERNAL_INDEX_FORMAT;; - boolean expectedIndexInternalFormat = randomBoolean(); - if (expectedIndexInternalFormat == false) { - indexFormat = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1), - randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT+1, Integer.MAX_VALUE)); - } - - Settings.Builder indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), indexFormat); - IndexMetaData.Builder metaDataBuilder = IndexMetaData.builder(Watch.INDEX).settings(indexSettings); - MetaData metaData = MetaData.builder().put(metaDataBuilder).build(); - - boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX); - assertThat(indexInternalFormat, is(expectedIndexInternalFormat)); - } - - public void testInternalIndexFormatCheckOnNonExistingIndex() { - MetaData metaData = MetaData.builder().build(); - boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX); - assertThat(indexInternalFormat, is(true)); - } - private static DiscoveryNode newNode(String nodeName) { return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);