Improve upgrade experience of node level index settings

In 5.0 we don't allow index settings to be specified on the node level ie.
in yaml files or via commandline argument. This can cause problems during
upgrade if this was used extensively. For instance if analyzers where
specified on a node level this might cause the index to be closed when
imported (see #17187). In such a case all indices relying on this
must be updated via `PUT /${index}/_settings`. Yet, this API has slightly
different semantics since it overrides existing settings. To make this less
painful this change adds a `preserve_existing` parameter on that API to ensure
we have the same semantics as if the setting was applied on the node level.

This change also adds a better error message and a change to the migration guide
to ensure upgrades are smooth if index settings are specified on the node level.

If a index setting is detected this change fails the node startup and prints a message
like this:
```
*************************************************************************************
Found index level settings on node level configuration.

Since elasticsearch 5.x index level settings can NOT be set on the nodes
configuration like the elasticsearch.yaml, in system properties or command line
arguments.In order to upgrade all indices the settings must be updated via the
/${index}/_settings API. Unless all settings are dynamic all indices must be closed
in order to apply the upgradeIndices created in the future should use index templates
to set default values.

Please ensure all required values are updated on all indices by executing:

curl -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '{
  "index.number_of_shards" : "1",
  "index.query.default_field" : "main_field",
  "index.translog.durability" : "async",
  "index.ttl.disable_purge" : "true"
}'
*************************************************************************************
```

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

@@ -80,6 +80,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
         UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
                 .indices(concreteIndices)
                 .settings(request.settings())
+                .setPreserveExisting(request.isPreserveExisting())
                 .ackTimeout(request.timeout())
                 .masterNodeTimeout(request.masterNodeTimeout());
 

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java

@@ -29,8 +29,23 @@ public class UpdateSettingsClusterStateUpdateRequest extends IndicesClusterState
 
     private Settings settings;
 
-    public UpdateSettingsClusterStateUpdateRequest() {
+    private boolean preserveExisting = false;
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsClusterStateUpdateRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
     }
 
     /**

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java

@@ -47,6 +47,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     private String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);
     private Settings settings = EMPTY_SETTINGS;
+    private boolean preserveExisting = false;
 
     public UpdateSettingsRequest() {
     }
@@ -127,6 +128,23 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         return this;
     }
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
+    }
+
     /**
      * Sets the settings to be updated (either json/yaml/properties format)
      */
@@ -149,6 +167,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         settings = readSettingsFromStream(in);
         readTimeout(in);
+        preserveExisting = in.readBoolean();
     }
 
     @Override
@@ -158,5 +177,6 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         indicesOptions.writeIndicesOptions(out);
         writeSettingsToStream(settings, out);
         writeTimeout(out);
+        out.writeBoolean(preserveExisting);
     }
 }

core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java

@@ -84,4 +84,9 @@ public class UpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<Upd
         request.settings(source);
         return this;
     }
+
+    public UpdateSettingsRequestBuilder setPreserveExisting(boolean preserveExisting) {
+        request.setPreserveExisting(preserveExisting);
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -176,6 +176,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         final Settings skippedSettigns = skipppedSettings.build();
         final Settings closedSettings = settingsForClosedIndices.build();
         final Settings openSettings = settingsForOpenIndices.build();
+        final boolean preserveExisting = request.isPreserveExisting();
 
         clusterService.submitStateUpdateTask("update-settings",
                 new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
@@ -221,7 +222,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                 }
 
                 int updatedNumberOfReplicas = openSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
-                if (updatedNumberOfReplicas != -1) {
+                if (updatedNumberOfReplicas != -1 && preserveExisting == false) {
                     routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
@@ -239,6 +240,9 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                         if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }
@@ -250,6 +254,9 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                         if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }

core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java

@@ -30,6 +30,9 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
@@ -221,10 +224,18 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
      * * Validates that all given settings are registered and valid
      */
     public final void validate(Settings settings) {
-        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
+        List<RuntimeException> exceptions = new ArrayList<>();
+        // we want them sorted for deterministic error messages
+        SortedMap<String, String> sortedSettings = new TreeMap<>(settings.getAsMap());
+        for (Map.Entry<String, String> entry : sortedSettings.entrySet()) {
+            try {
                 validate(entry.getKey(), settings);
+            } catch (RuntimeException ex) {
+                exceptions.add(ex);
             }
         }
+        ExceptionsHelper.rethrowAndSuppress(exceptions);
+    }
 
 
     /**

core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java

@@ -20,13 +20,22 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tribe.TribeService;
 
+import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * A module that binds the provided settings to the {@link Settings} interface.
@@ -37,9 +46,12 @@ public class SettingsModule extends AbstractModule {
     private final Set<String> settingsFilterPattern = new HashSet<>();
     private final Map<String, Setting<?>> nodeSettings = new HashMap<>();
     private final Map<String, Setting<?>> indexSettings = new HashMap<>();
-    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.") && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.")
+        && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private final ESLogger logger;
 
     public SettingsModule(Settings settings) {
+        logger = Loggers.getLogger(getClass(), settings);
         this.settings = settings;
         for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
             registerSetting(setting);
@@ -53,6 +65,56 @@ public class SettingsModule extends AbstractModule {
     protected void configure() {
         final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
         final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
+        Settings indexSettings = settings.filter((s) -> s.startsWith("index.") && clusterSettings.get(s) == null);
+        if (indexSettings.isEmpty() == false) {
+            try {
+                String separator = IntStream.range(0, 85).mapToObj(s -> "*").collect(Collectors.joining("")).trim();
+                StringBuilder builder = new StringBuilder();
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+                builder.append("Found index level settings on node level configuration.");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                int count = 0;
+                for (String word : ("Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like " +
+                    "the elasticsearch.yaml, in system properties or command line arguments." +
+                    "In order to upgrade all indices the settings must be updated via the /${index}/_settings API. " +
+                    "Unless all settings are dynamic all indices must be closed in order to apply the upgrade" +
+                    "Indices created in the future should use index templates to set default values."
+                    ).split(" ")) {
+                    if (count + word.length() > 85) {
+                        builder.append(System.lineSeparator());
+                        count = 0;
+                    }
+                    count += word.length() + 1;
+                    builder.append(word).append(" ");
+                }
+
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("Please ensure all required values are updated on all indices by executing: ");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("curl -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '");
+                try (XContentBuilder xContentBuilder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    xContentBuilder.prettyPrint();
+                    xContentBuilder.startObject();
+                    indexSettings.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                    xContentBuilder.endObject();
+                    builder.append(xContentBuilder.string());
+                }
+                builder.append("'");
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+
+                logger.warn(builder.toString());
+                throw new IllegalArgumentException("node settings must not contain any index level settings");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
         // by now we are fully configured, lets check node level settings for unregistered index settings
         final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));

core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java

@@ -47,6 +47,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
             "timeout",
             "master_timeout",
             "index",
+            "preserve_existing",
             "expand_wildcards",
             "ignore_unavailable",
             "allow_no_indices"));
@@ -62,6 +63,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
         UpdateSettingsRequest updateSettingsRequest = updateSettingsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         updateSettingsRequest.timeout(request.paramAsTime("timeout", updateSettingsRequest.timeout()));
+        updateSettingsRequest.setPreserveExisting(request.paramAsBoolean("preserve_existing", updateSettingsRequest.isPreserveExisting()));
         updateSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", updateSettingsRequest.masterNodeTimeout()));
         updateSettingsRequest.indicesOptions(IndicesOptions.fromRequest(request, updateSettingsRequest.indicesOptions()));
 

core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java

@@ -36,12 +36,35 @@ public class SettingsModuleTests extends ModuleTestCase {
         {
             Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]").build();
             SettingsModule module = new SettingsModule(settings);
-            try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
-                fail();
-            } catch (IllegalArgumentException ex) {
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
             assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
         }
+
+        {
+            Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]")
+                .put("some.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
+            assertEquals(1, ex.getSuppressed().length);
+            assertEquals("unknown setting [some.foo.bar]", ex.getSuppressed()[0].getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default")
+                .put("index.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("node settings must not contain any index level settings", ex.getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default").build();
+            SettingsModule module = new SettingsModule(settings);
+            assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         }
     }
 

docs/reference/migration/migrate_5_0/settings.asciidoc

@@ -12,6 +12,15 @@ that plugins that define custom settings must register all of their settings
 during plugin loading using the `SettingsModule#registerSettings(Setting)`
 method.
 
+==== Index Level Settings
+
+In previous versions Elasticsearch allowed to specify index level setting
+as _defaults_ on the node level, inside the `elasticsearch.yaml` file or even via
+command-line parameters. From Elasticsearch 5.0 on only selected settings like
+for instance `index.codec` can be set on the node level. All other settings must be
+set on each individual index. To set default values on every index, index templates
+should be used instead.
+
 ==== Node settings
 
 The `name` setting has been removed and is replaced by `node.name`. Usage of

rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json

@@ -16,6 +16,10 @@
           "type": "time",
           "description": "Specify timeout for connection to master"
         },
+        "preserve_existing": {
+          "type": "boolean",
+          "description": "Whether to update existing settings. If set to `true` existing settings on an index remain unchanged, the default is `false`"
+        },
         "ignore_unavailable": {
           "type": "boolean",
           "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed)"

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml

@@ -54,3 +54,70 @@ setup:
         body:
           number_of_replicas: 1
 
+---
+"Test preserve_existing settings":
+  - do:
+      indices.put_settings:
+        index: test-index
+        body:
+          number_of_replicas: 0
+
+  - do:
+      indices.get_settings:
+        flat_settings: false
+
+  - match:
+      test-index.settings.index.number_of_replicas: "0"
+
+  - do:
+      indices.put_settings:
+        preserve_existing: true
+        index: test-index
+        body:
+          index.number_of_replicas: 1
+          index.translog.durability: "async"
+
+  - do:
+      indices.get_settings:
+        flat_settings: false
+
+
+  - match:
+      test-index.settings.index.number_of_replicas: "0"
+  - match:
+      test-index.settings.index.translog.durability: "async"
+
+  - do:
+      indices.close:
+        index: test-index
+
+  - do:
+      indices.put_settings:
+        preserve_existing: true
+        index: test-index
+        body:
+          index.translog.durability: "request"
+          index.query_string.lenient: "true"
+
+  - do:
+      indices.get_settings:
+        index: test-index
+        flat_settings: false
+
+  - match:
+      test-index.settings.index.query_string.lenient: "true"
+  - match:
+      test-index.settings.index.translog.durability: "async"
+
+  - do:
+       indices.open:
+         index: test-index
+  - do:
+      indices.get_settings:
+        index: test-index
+        flat_settings: false
+
+  - match:
+     test-index.settings.index.query_string.lenient: "true"
+  - match:
+     test-index.settings.index.translog.durability: "async"