Disallow include_in_all for 6.0+ indices

Since `_all` is now deprecated and cannot be set for new indices, we should also
disallow any field that has the `include_in_all` parameter set.

Resolves #22923

core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java

@@ -247,6 +247,10 @@ public class TypeParsers {
                 if (parserContext.isWithinMultiField()) {
                     throw new MapperParsingException("include_in_all in multi fields is not allowed. Found the include_in_all in field ["
                         + name + "] which is within a multi field.");
+                } else if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                    throw new MapperParsingException("[include_in_all] is not allowed for indices created on or after version 6.0.0 as " +
+                                    "[_all] is deprecated. As a replacement, you can use an [copy_to] on mapping fields to create your " +
+                                    "own catch all field.");
                 } else {
                     builder.includeInAll(nodeBooleanValue(name, "include_in_all", propNode, parserContext));
                 }

core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java

@@ -45,12 +45,4 @@ public class BWCTemplateTests extends ESSingleNodeTestCase {
         client().prepareIndex("winlogbeat-foo", "doc", "1").setSource("message", "foo").get();
         assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
     }
-
-    public void testLogstashTemplatesBWC() throws Exception {
-        byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json");
-        client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get();
-        client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
-        assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
-    }
-
 }

core/src/test/java/org/elasticsearch/index/mapper/MapperTests.java

@@ -21,9 +21,15 @@ package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
 public class MapperTests extends ESTestCase {
 
     public void testSuccessfulBuilderContext() {
@@ -39,5 +45,37 @@ public class MapperTests extends ESTestCase {
         NullPointerException e = expectThrows(NullPointerException.class, () -> new Mapper.BuilderContext(null, new ContentPath(1)));
     }
 
+    public void testExceptionForIncludeInAll() throws IOException {
+        XContentBuilder mapping = createMappingWithIncludeInAll();
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+
+        final MapperService currentMapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings);
+        Exception e = expectThrows(MapperParsingException.class, () ->
+                currentMapperService.parse("type", new CompressedXContent(mapping.string()), true));
+        assertEquals("[include_in_all] is not allowed for indices created on or after version 6.0.0 as [_all] is deprecated. " +
+                        "As a replacement, you can use an [copy_to] on mapping fields to create your own catch all field.",
+                e.getMessage());
+
+        settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_3_0_UNRELEASED).build();
+
+        // Create the mapping service with an older index creation version
+        final MapperService oldMapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings);
+        // Should not throw an exception now
+        oldMapperService.parse("type", new CompressedXContent(mapping.string()), true);
+    }
+
+    private static XContentBuilder createMappingWithIncludeInAll() throws IOException {
+        return jsonBuilder()
+                .startObject()
+                .startObject("type")
+                .startObject("properties")
+                .startObject("a")
+                .field("type", "text")
+                .field("include_in_all", randomBoolean())
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+    }
 
 }

core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json

@@ -1,45 +0,0 @@
-{
-  "template" : "logstash-*",
-  "version" : 50001,
-  "settings" : {
-    "index.refresh_interval" : "5s"
-  },
-  "mappings" : {
-    "_default_" : {
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword" }
-            }
-          }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "include_in_all": false },
-        "@version": { "type": "keyword", "include_in_all": false },
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
-        }
-      }
-    }
-  }
-}

docs/reference/mapping.asciidoc

@@ -133,10 +133,10 @@ are not independent of each other. Fields with:
 * and *must have the same mapping*.
 
 If a `title` field exists in both the `user` and `blogpost` mapping types, the
-`title` fields must have exactly the same mapping in each type.  The only
+`title` fields must have exactly the same mapping in each type. The only
 exceptions to this rule are the <<copy-to>>, <<dynamic>>, <<enabled>>,
-<<ignore-above>>, <<include-in-all>>, and <<properties>> parameters, which may
+<<ignore-above>>, and <<properties>> parameters, which may have different
-have different settings per field.
+settings per field.
 
 Usually, fields with the same name also contain the same type of data, so
 having the same mapping is not a problem.  When conflicts do arise, these can

docs/reference/mapping/fields/all-field.asciidoc

@@ -71,9 +71,8 @@ and long fields (less relevant). For use cases where search relevance is
 important, it is better to query individual fields specifically.
 
 The `_all` field is not free: it requires extra CPU cycles and uses more disk
-space. For this reason, it is disabled by default. If not needed, it can be
+space. For this reason, it is disabled by default. If needed, it can be
-<<enabling-all-field,enabled>> or customised on a <<include-in-all,per-field
+<<enabling-all-field,enabled>>.
-basis>>.
 
 [[querying-all-field]]
 ==== Using the `_all` field in queries
@@ -163,13 +162,6 @@ PUT my_index
 
 <1> The `query_string` query will default to querying the `content` field in this index.
 
-[[excluding-from-all]]
-==== Excluding fields from `_all`
-
-Individual fields can be included or excluded from the `_all` field with the
-<<include-in-all,`include_in_all`>> setting.
-
-
 [[all-field-and-boosting]]
 ==== Index boosting and the `_all` field
 

docs/reference/mapping/params.asciidoc

@@ -19,7 +19,6 @@ The following mapping parameters are common to some or all field datatypes:
 * <<mapping-date-format,`format`>>
 * <<ignore-above,`ignore_above`>>
 * <<ignore-malformed,`ignore_malformed`>>
-* <<include-in-all,`include_in_all`>>
 * <<index-options,`index_options`>>
 * <<mapping-index,`index`>>
 * <<multi-fields,`fields`>>
@@ -57,8 +56,6 @@ include::params/ignore-above.asciidoc[]
 
 include::params/ignore-malformed.asciidoc[]
 
-include::params/include-in-all.asciidoc[]
-
 include::params/index.asciidoc[]
 
 include::params/index-options.asciidoc[]

docs/reference/mapping/params/include-in-all.asciidoc

@@ -1,88 +0,0 @@
-[[include-in-all]]
-=== `include_in_all`
-
-The `include_in_all` parameter provides per-field control over which fields
-are included in the <<mapping-all-field,`_all`>> field.  It defaults to `true`, unless <<mapping-index,`index`>> is set to `no`.
-
-This example demonstrates how to exclude the `date` field from the `_all` field:
-
-[source,js]
---------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": { <1>
-          "type": "text"
-        },
-        "content": { <1>
-          "type": "text"
-        },
-        "date": { <2>
-          "type": "date",
-          "include_in_all": false
-        }
-      }
-    }
-  }
-}
---------------------------------
-// CONSOLE
-
-<1> The `title` and `content` fields will be included in the `_all` field.
-<2> The `date` field will not be included in the `_all` field.
-
-TIP: The `include_in_all` setting is allowed to have different settings for
-fields of the same name in the same index.  Its value can be updated on
-existing fields using the <<indices-put-mapping,PUT mapping API>>.
-
-
-The `include_in_all` parameter can also be set at the type level and on
-<<object,`object`>> or <<nested,`nested`>> fields, in which case all sub-
-fields inherit that setting.  For instance:
-
-[source,js]
---------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "include_in_all": false, <1>
-      "properties": {
-        "title":          { "type": "text" },
-        "author": {
-          "include_in_all": true, <2>
-          "properties": {
-            "first_name": { "type": "text" },
-            "last_name":  { "type": "text" }
-          }
-        },
-        "editor": {
-          "properties": {
-            "first_name": { "type": "text" }, <3>
-            "last_name":  { "type": "text", "include_in_all": true } <3>
-          }
-        }
-      }
-    }
-  }
-}
---------------------------------
-// CONSOLE
-
-<1> All fields in `my_type` are excluded from `_all`.
-<2> The `author.first_name` and `author.last_name` fields are included in `_all`.
-<3> Only the `editor.last_name` field is included in `_all`.
-    The `editor.first_name` inherits the type-level setting and is excluded.
-
-[NOTE]
-.Multi-fields and `include_in_all`
-=================================
-
-The original field value is added to the `_all` field, not the terms produced
-by a field's analyzer.  For this reason, it makes no sense to set
-`include_in_all` to `true` on <<multi-fields,multi-fields>>, as each
-multi-field has exactly the same value as its parent.
-
-=================================

docs/reference/mapping/types/date.asciidoc

@@ -117,14 +117,6 @@ The following parameters are accepted by `date` fields:
     If `true`, malformed numbers are ignored. If `false` (default), malformed
     numbers throw an exception and reject the whole document.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) and `false`.

docs/reference/mapping/types/ip.asciidoc

@@ -54,14 +54,6 @@ The following parameters are accepted by `ip` fields:
     can later be used for sorting, aggregations, or scripting? Accepts `true`
     (default) or `false`.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) and `false`.

docs/reference/mapping/types/keyword.asciidoc

@@ -65,14 +65,6 @@ The following parameters are accepted by `keyword` fields:
     Do not index any string longer than this value.  Defaults to
     `2147483647` so that all values would be accepted.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) or `false`.

docs/reference/mapping/types/nested.asciidoc

@@ -171,13 +171,6 @@ The following parameters are accepted by `nested` fields:
     Whether or not new `properties` should be added dynamically to an existing
     nested object.  Accepts `true` (default), `false` and `strict`.
 
-<<include-in-all,`include_in_all`>>::
-
-    Sets the default `include_in_all` value for all the `properties` within
-    the nested object. Nested documents do not have their own `_all` field.
-    Instead, values are added to the `_all` field of the main ``root''
-    document.
-
 <<properties,`properties`>>::
 
     The fields within the nested object, which can be of any

docs/reference/mapping/types/numeric.asciidoc

@@ -108,14 +108,6 @@ The following parameters are accepted by numeric types:
     If `true`, malformed numbers are ignored. If `false` (default), malformed
     numbers throw an exception and reject the whole document.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) and `false`.

docs/reference/mapping/types/object.asciidoc

@@ -88,11 +88,6 @@ The following parameters are accepted by `object` fields:
     Whether the JSON value given for the object field should be
     parsed and indexed (`true`, default) or completely ignored (`false`).
 
-<<include-in-all,`include_in_all`>>::
-
-    Sets the default `include_in_all` value for all the `properties` within
-    the object. The object itself is not added to the `_all` field.
-
 <<properties,`properties`>>::
 
     The fields within the object, which can be of any

docs/reference/mapping/types/range.asciidoc

@@ -126,14 +126,6 @@ The following parameters are accepted by range types:
     Mapping field-level query time boosting. Accepts a floating point number, defaults
     to `1.0`.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) and `false`.

docs/reference/mapping/types/text.asciidoc

@@ -80,14 +80,6 @@ The following parameters are accepted by `text` fields:
     sorting and aggregations, or the same string value analyzed by different
     analyzers.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
 <<mapping-index,`index`>>::
 
     Should the field be searchable? Accepts `true` (default) or `false`.

docs/reference/mapping/types/token-count.asciidoc

@@ -83,13 +83,6 @@ The following parameters are accepted by `token_count` fields:
 
     Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
 
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false`. Note: if `true`, it is the string value that is added to `_all`,
-    not the calculated token count.
-
 <<null-value,`null_value`>>::
 
     Accepts a numeric value of the same `type` as the field which is

docs/reference/migration/migrate_6_0/mappings.asciidoc

@@ -18,6 +18,12 @@ check if `_all` is enabled/disabled and switch to executing the query across all
 fields if `_all` is disabled. `_all` can no longer be configured for indices
 created with Elasticsearch version 6.0 or later.
 
+==== The `include_in_all` mapping parameter is now disallowed
+
+Since the `_all` field is now disabled by default and cannot be configured for
+indices created with Elasticsearch 6.0 or later, the `include_in_all` setting is
+now disallowed for these indices' mappings.
+
 ==== Unrecognized `match_mapping_type` options not silently ignored
 
 Previously Elastiscearch would silently ignore any dynamic templates that