Revert "X-Pack: Add index.internal.format index setting (elastic/x-pack-elasticsearch#1700)"

This reverts commit elastic/x-pack-elasticsearch@e430691c51.

The setting is going to be applied in core, so that only parts
of these changes need to be applied here (in a seperate PR).

Original commit: elastic/x-pack-elasticsearch@9e178c87fa

plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -46,6 +46,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
@@ -106,7 +107,11 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.security.AccessController;
 import java.security.GeneralSecurityException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedAction;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -146,11 +151,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
     // inside of YAML settings we still use xpack do not having handle issues with dashes
     private static final String SETTINGS_NAME = "xpack";
 
-    // internal index format is used for upgrading
-    public static final Setting<Integer> INDEX_INTERNAL_FORMAT_SETTING =
-            Setting.intSetting("index.internal.format", 0, Setting.Property.IndexScope);
-    public static final int INTERNAL_INDEX_FORMAT = 6;
-
     // TODO: clean up this library to not ask for write access to all system properties!
     static {
         // invoke this clinit in unbound with permissions to access all system properties
@@ -359,8 +359,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
         settings.addAll(licensing.getSettings());
         settings.addAll(XPackSettings.getAllSettings());
 
-        // an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
+        // we add the `xpack.version` setting to all internal indices
-        settings.add(INDEX_INTERNAL_FORMAT_SETTING);
+        settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope));
 
         // notification services
         settings.add(SlackService.SLACK_ACCOUNT_SETTING);

plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java

@@ -9,7 +9,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -20,8 +19,6 @@ import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.XPackPlugin;
-import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
 import org.elasticsearch.xpack.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils;
 
@@ -95,10 +92,8 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
         }
     }
 
-    /**
+    /*
-     *
+     * TODO possible optimization
-     * @param event The event of containing the new cluster state
-     *
      * stop certain parts of watcher, when there are no watcher indices on this node by checking the shardrouting
      * note that this is not easily possible, because of the execute watch api, that needs to be able to execute anywhere!
      * this means, only certain components can be stopped
@@ -126,7 +121,6 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
                 DiscoveryNode localNode = event.state().nodes().getLocalNode();
                 RoutingNode routingNode = event.state().getRoutingNodes().node(localNode.getId());
                 IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(Watch.INDEX, event.state().metaData());
-
                 // no watcher index, time to pause, if we currently have shards here
                 if (watcherIndexMetaData == null) {
                     if (previousAllocationIds.get().isEmpty() == false) {
@@ -157,33 +151,11 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
                     executor.execute(() -> watcherService.reload(event.state(), "different shard allocation ids"));
                 }
             } else if (watcherService.state() != WatcherState.STARTED && watcherService.state() != WatcherState.STARTING) {
-                if (isIndexInternalFormat(event.state().metaData(), Watch.INDEX) &&
+                executor.execute(() -> start(event.state(), false));
-                        isIndexInternalFormat(event.state().metaData(), TriggeredWatchStore.INDEX_NAME)) {
-                    executor.execute(() -> start(event.state(), false));
-                } else {
-                    logger.error("Not starting watcher, the indices have not been upgraded yet. Please run the Upgrade API");
-                }
             }
         }
     }
 
-    /**
-     * If the supplied index exists, ensure that the 'index.internal.format' setting is configured appropriately.
-     * This ensures that watcher can use this index as needed.
-     *
-     * @param metaData  The cluster state meta data
-     * @param index     The index name to check for. Aliases will be resolved properly.
-     * @return          true if the index does not exist or the internal format is set properly
-     */
-    boolean isIndexInternalFormat(MetaData metaData, String index) {
-        IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(index, metaData);
-        if (watcherIndexMetaData == null) {
-            return true;
-        } else {
-            return XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.get(watcherIndexMetaData.getSettings()) == XPackPlugin.INTERNAL_INDEX_FORMAT;
-        }
-    }
-
     public WatcherMetaData watcherMetaData() {
         return watcherMetaData;
     }

plugin/src/main/resources/monitoring-alerts.json

@@ -5,7 +5,6 @@
     "index": {
       "number_of_shards": 1,
       "number_of_replicas": 1,
-      "internal.format": 6,
       "codec": "best_compression"
     }
   },

plugin/src/main/resources/monitoring-beats.json

@@ -4,7 +4,6 @@
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
-    "index.internal.format": 6,
     "index.codec": "best_compression"
   },
   "mappings": {

plugin/src/main/resources/monitoring-es.json

@@ -4,8 +4,7 @@
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
-    "index.codec": "best_compression",
+    "index.codec": "best_compression"
-    "index.internal.format": 6
   },
   "mappings": {
     "doc": {

plugin/src/main/resources/monitoring-kibana.json

@@ -4,8 +4,7 @@
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
-    "index.codec": "best_compression",
+    "index.codec": "best_compression"
-    "index.internal.format": 6
   },
   "mappings": {
     "doc": {

plugin/src/main/resources/monitoring-logstash.json

@@ -4,8 +4,7 @@
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
-    "index.codec": "best_compression",
+    "index.codec": "best_compression"
-    "index.internal.format": 6
   },
   "mappings": {
     "doc": {

plugin/src/main/resources/security-index-template.json

@@ -7,7 +7,6 @@
     "number_of_replicas" : 0,
     "auto_expand_replicas" : "0-all",
     "index.priority": 1000,
-    "index.internal.format": 6,
     "analysis" : {
       "filter" : {
         "email" : {

plugin/src/main/resources/security_audit_log.json

@@ -2,8 +2,7 @@
   "index_patterns": ".security_audit_log*",
   "order": 2147483647,
   "settings": {
-    "index.mapper.dynamic" : false,
+    "index.mapper.dynamic" : false
-    "index.internal.format": 6
   },
   "mappings": {
     "event": {

plugin/src/main/resources/triggered-watches.json

@@ -5,8 +5,7 @@
     "index.number_of_shards": 1,
     "index.mapper.dynamic" : false,
     "index.refresh_interval" : "-1",
-    "index.priority": 900,
+    "index.priority": 900
-    "index.internal.format": 6
   },
   "mappings": {
     "doc": {

plugin/src/main/resources/watch-history.json

@@ -3,7 +3,6 @@
   "order": 2147483647,
   "settings": {
     "xpack.watcher.template.version": "${xpack.watcher.template.version}",
-    "index.internal.format": 6,
     "index.number_of_shards": 1,
     "index.mapper.dynamic": false
   },

plugin/src/main/resources/watches.json

@@ -4,8 +4,7 @@
   "settings": {
     "index.number_of_shards": 1,
     "index.mapper.dynamic" : false,
-    "index.priority": 800,
+    "index.priority": 800
-    "index.internal.format": 6
   },
   "mappings": {
     "doc": {

plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java

@@ -28,8 +28,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.XPackPlugin;
-import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
 import org.elasticsearch.xpack.watcher.watch.Watch;
 import org.junit.Before;
 import org.mockito.stubbing.Answer;
@@ -41,7 +39,6 @@ import java.util.concurrent.ExecutorService;
 import static java.util.Arrays.asList;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
-import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.anyString;
@@ -355,65 +352,6 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
         verify(watcherService, times(1)).pauseExecution(anyObject());
     }
 
-    public void testWatcherDoesNotStartWithOldIndexFormat() throws Exception {
-        String index = randomFrom(Watch.INDEX, TriggeredWatchStore.INDEX_NAME);
-        Index watchIndex = new Index(index, "foo");
-        ShardId shardId = new ShardId(watchIndex, 0);
-        IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(watchIndex)
-                .addShard(TestShardRouting.newShardRouting(shardId, "node_1", true, STARTED)).build();
-        DiscoveryNodes nodes = new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")).build();
-
-        Settings.Builder indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
-        if (randomBoolean()) {
-            int randomNumber = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
-                    randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT, Integer.MAX_VALUE));
-            indexSettings.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), randomNumber);
-        }
-        IndexMetaData.Builder newIndexMetaDataBuilder = IndexMetaData.builder(index).settings(indexSettings);
-
-        ClusterState clusterStateWithWatcherIndex = ClusterState.builder(new ClusterName("my-cluster"))
-                .nodes(nodes)
-                .routingTable(RoutingTable.builder().add(watchRoutingTable).build())
-                .metaData(MetaData.builder().put(newIndexMetaDataBuilder))
-                .build();
-
-        ClusterState emptyClusterState = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build();
-
-        when(watcherService.state()).thenReturn(WatcherState.STOPPED);
-        when(watcherService.validate(eq(clusterStateWithWatcherIndex))).thenReturn(true);
-        lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterStateWithWatcherIndex, emptyClusterState));
-        verify(watcherService, never()).start(any(ClusterState.class));
-    }
-
-    public void testInternalIndexFormatCheck() {
-        int indexFormat = XPackPlugin.INTERNAL_INDEX_FORMAT;;
-        boolean expectedIndexInternalFormat = randomBoolean();
-        if (expectedIndexInternalFormat == false) {
-            indexFormat = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
-                    randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT+1, Integer.MAX_VALUE));
-        }
-
-        Settings.Builder indexSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), indexFormat);
-        IndexMetaData.Builder metaDataBuilder = IndexMetaData.builder(Watch.INDEX).settings(indexSettings);
-        MetaData metaData = MetaData.builder().put(metaDataBuilder).build();
-
-        boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
-        assertThat(indexInternalFormat, is(expectedIndexInternalFormat));
-    }
-
-    public void testInternalIndexFormatCheckOnNonExistingIndex() {
-        MetaData metaData = MetaData.builder().build();
-        boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
-        assertThat(indexInternalFormat, is(true));
-    }
-
     private static DiscoveryNode newNode(String nodeName) {
         return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
                 new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);