honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Disallow include_in_all for 6.0+ indices 

Since `_all` is now deprecated and cannot be set for new indices, we should also
disallow any field that has the `include_in_all` parameter set.

Resolves #22923
This commit is contained in:
Lee Hinman 2017-02-03 13:38:59 -07:00
parent ad4bfa2307
commit b3c27a7fdd
18 changed files with 53 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
View File

@ -247,6 +247,10 @@ public class TypeParsers {
if (parserContext.isWithinMultiField()) {
throw new MapperParsingException("include_in_all in multi fields is not allowed. Found the include_in_all in field ["
+ name + "] which is within a multi field.");
} else if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
throw new MapperParsingException("[include_in_all] is not allowed for indices created on or after version 6.0.0 as " +
"[_all] is deprecated. As a replacement, you can use an [copy_to] on mapping fields to create your " +
"own catch all field.");
} else {
builder.includeInAll(nodeBooleanValue(name, "include_in_all", propNode, parserContext));
}

8
core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
View File

@ -45,12 +45,4 @@ public class BWCTemplateTests extends ESSingleNodeTestCase {
client().prepareIndex("winlogbeat-foo", "doc", "1").setSource("message", "foo").get();
assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
}
public void testLogstashTemplatesBWC() throws Exception {
byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json");
client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get();
client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
}
}

38
core/src/test/java/org/elasticsearch/index/mapper/MapperTests.java
View File

@ -21,9 +21,15 @@ package org.elasticsearch.index.mapper;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
public class MapperTests extends ESTestCase {
public void testSuccessfulBuilderContext() {
@ -39,5 +45,37 @@ public class MapperTests extends ESTestCase {
NullPointerException e = expectThrows(NullPointerException.class, () -> new Mapper.BuilderContext(null, new ContentPath(1)));
}
public void testExceptionForIncludeInAll() throws IOException {
XContentBuilder mapping = createMappingWithIncludeInAll();
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
final MapperService currentMapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings);
Exception e = expectThrows(MapperParsingException.class, () ->
currentMapperService.parse("type", new CompressedXContent(mapping.string()), true));
assertEquals("[include_in_all] is not allowed for indices created on or after version 6.0.0 as [_all] is deprecated. " +
"As a replacement, you can use an [copy_to] on mapping fields to create your own catch all field.",
e.getMessage());
settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_3_0_UNRELEASED).build();
// Create the mapping service with an older index creation version
final MapperService oldMapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings);
// Should not throw an exception now
oldMapperService.parse("type", new CompressedXContent(mapping.string()), true);
}
private static XContentBuilder createMappingWithIncludeInAll() throws IOException {
return jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("a")
.field("type", "text")
.field("include_in_all", randomBoolean())
.endObject()
.endObject()
.endObject()
.endObject();
}
}

45
core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json
View File

@ -1,45 +0,0 @@
{
"template" : "logstash-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword" }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date", "include_in_all": false },
"@version": { "type": "keyword", "include_in_all": false },
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}

6
docs/reference/mapping.asciidoc
View File

@ -133,10 +133,10 @@ are not independent of each other. Fields with:
* and *must have the same mapping*.
If a `title` field exists in both the `user` and `blogpost` mapping types, the
`title` fields must have exactly the same mapping in each type. The only
`title` fields must have exactly the same mapping in each type. The only
exceptions to this rule are the <<copy-to>>, <<dynamic>>, <<enabled>>,
<<ignore-above>>, <<include-in-all>>, and <<properties>> parameters, which may
have different settings per field.
<<ignore-above>>, and <<properties>> parameters, which may have different
settings per field.
Usually, fields with the same name also contain the same type of data, so
having the same mapping is not a problem. When conflicts do arise, these can

12
docs/reference/mapping/fields/all-field.asciidoc
View File

@ -71,9 +71,8 @@ and long fields (less relevant). For use cases where search relevance is
important, it is better to query individual fields specifically.
The `_all` field is not free: it requires extra CPU cycles and uses more disk
space. For this reason, it is disabled by default. If not needed, it can be
<<enabling-all-field,enabled>> or customised on a <<include-in-all,per-field
basis>>.
space. For this reason, it is disabled by default. If needed, it can be
<<enabling-all-field,enabled>>.
[[querying-all-field]]
==== Using the `_all` field in queries
@ -163,13 +162,6 @@ PUT my_index
<1> The `query_string` query will default to querying the `content` field in this index.
[[excluding-from-all]]
==== Excluding fields from `_all`
Individual fields can be included or excluded from the `_all` field with the
<<include-in-all,`include_in_all`>> setting.
[[all-field-and-boosting]]
==== Index boosting and the `_all` field

3
docs/reference/mapping/params.asciidoc
View File

@ -19,7 +19,6 @@ The following mapping parameters are common to some or all field datatypes:
* <<mapping-date-format,`format`>>
* <<ignore-above,`ignore_above`>>
* <<ignore-malformed,`ignore_malformed`>>
* <<include-in-all,`include_in_all`>>
* <<index-options,`index_options`>>
* <<mapping-index,`index`>>
* <<multi-fields,`fields`>>
@ -57,8 +56,6 @@ include::params/ignore-above.asciidoc[]
include::params/ignore-malformed.asciidoc[]
include::params/include-in-all.asciidoc[]
include::params/index.asciidoc[]
include::params/index-options.asciidoc[]

88
docs/reference/mapping/params/include-in-all.asciidoc
View File

@ -1,88 +0,0 @@
[[include-in-all]]
=== `include_in_all`
The `include_in_all` parameter provides per-field control over which fields
are included in the <<mapping-all-field,`_all`>> field. It defaults to `true`, unless <<mapping-index,`index`>> is set to `no`.
This example demonstrates how to exclude the `date` field from the `_all` field:
[source,js]
--------------------------------
PUT my_index
{
"mappings": {
"my_type": {
"properties": {
"title": { <1>
"type": "text"
},
"content": { <1>
"type": "text"
},
"date": { <2>
"type": "date",
"include_in_all": false
}
}
}
}
}
--------------------------------
// CONSOLE
<1> The `title` and `content` fields will be included in the `_all` field.
<2> The `date` field will not be included in the `_all` field.
TIP: The `include_in_all` setting is allowed to have different settings for
fields of the same name in the same index. Its value can be updated on
existing fields using the <<indices-put-mapping,PUT mapping API>>.
The `include_in_all` parameter can also be set at the type level and on
<<object,`object`>> or <<nested,`nested`>> fields, in which case all sub-
fields inherit that setting. For instance:
[source,js]
--------------------------------
PUT my_index
{
"mappings": {
"my_type": {
"include_in_all": false, <1>
"properties": {
"title": { "type": "text" },
"author": {
"include_in_all": true, <2>
"properties": {
"first_name": { "type": "text" },
"last_name": { "type": "text" }
}
},
"editor": {
"properties": {
"first_name": { "type": "text" }, <3>
"last_name": { "type": "text", "include_in_all": true } <3>
}
}
}
}
}
}
--------------------------------
// CONSOLE
<1> All fields in `my_type` are excluded from `_all`.
<2> The `author.first_name` and `author.last_name` fields are included in `_all`.
<3> Only the `editor.last_name` field is included in `_all`.
The `editor.first_name` inherits the type-level setting and is excluded.
[NOTE]
.Multi-fields and `include_in_all`
=================================
The original field value is added to the `_all` field, not the terms produced
by a field's analyzer. For this reason, it makes no sense to set
`include_in_all` to `true` on <<multi-fields,multi-fields>>, as each
multi-field has exactly the same value as its parent.
=================================

8
docs/reference/mapping/types/date.asciidoc
View File

@ -117,14 +117,6 @@ The following parameters are accepted by `date` fields:
If `true`, malformed numbers are ignored. If `false` (default), malformed
numbers throw an exception and reject the whole document.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) and `false`.

8
docs/reference/mapping/types/ip.asciidoc
View File

@ -54,14 +54,6 @@ The following parameters are accepted by `ip` fields:
can later be used for sorting, aggregations, or scripting? Accepts `true`
(default) or `false`.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) and `false`.

8
docs/reference/mapping/types/keyword.asciidoc
View File

@ -65,14 +65,6 @@ The following parameters are accepted by `keyword` fields:
Do not index any string longer than this value. Defaults to
`2147483647` so that all values would be accepted.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) or `false`.

7
docs/reference/mapping/types/nested.asciidoc
View File

@ -171,13 +171,6 @@ The following parameters are accepted by `nested` fields:
Whether or not new `properties` should be added dynamically to an existing
nested object. Accepts `true` (default), `false` and `strict`.
<<include-in-all,`include_in_all`>>::
Sets the default `include_in_all` value for all the `properties` within
the nested object. Nested documents do not have their own `_all` field.
Instead, values are added to the `_all` field of the main ``root''
document.
<<properties,`properties`>>::
The fields within the nested object, which can be of any

8
docs/reference/mapping/types/numeric.asciidoc
View File

@ -108,14 +108,6 @@ The following parameters are accepted by numeric types:
If `true`, malformed numbers are ignored. If `false` (default), malformed
numbers throw an exception and reject the whole document.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) and `false`.

5
docs/reference/mapping/types/object.asciidoc
View File

@ -88,11 +88,6 @@ The following parameters are accepted by `object` fields:
Whether the JSON value given for the object field should be
parsed and indexed (`true`, default) or completely ignored (`false`).
<<include-in-all,`include_in_all`>>::
Sets the default `include_in_all` value for all the `properties` within
the object. The object itself is not added to the `_all` field.
<<properties,`properties`>>::
The fields within the object, which can be of any

8
docs/reference/mapping/types/range.asciidoc
View File

@ -126,14 +126,6 @@ The following parameters are accepted by range types:
Mapping field-level query time boosting. Accepts a floating point number, defaults
to `1.0`.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) and `false`.

8
docs/reference/mapping/types/text.asciidoc
View File

@ -80,14 +80,6 @@ The following parameters are accepted by `text` fields:
sorting and aggregations, or the same string value analyzed by different
analyzers.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
<<object,`object`>> field sets `include_in_all` to `false`.
Otherwise defaults to `true`.
<<mapping-index,`index`>>::
Should the field be searchable? Accepts `true` (default) or `false`.

7
docs/reference/mapping/types/token-count.asciidoc
View File

@ -83,13 +83,6 @@ The following parameters are accepted by `token_count` fields:
Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
<<include-in-all,`include_in_all`>>::
Whether or not the field value should be included in the
<<mapping-all-field,`_all`>> field? Accepts `true` or `false`. Defaults
to `false`. Note: if `true`, it is the string value that is added to `_all`,
not the calculated token count.
<<null-value,`null_value`>>::
Accepts a numeric value of the same `type` as the field which is

6
docs/reference/migration/migrate_6_0/mappings.asciidoc
View File

@ -18,6 +18,12 @@ check if `_all` is enabled/disabled and switch to executing the query across all
fields if `_all` is disabled. `_all` can no longer be configured for indices
created with Elasticsearch version 6.0 or later.
==== The `include_in_all` mapping parameter is now disallowed
Since the `_all` field is now disabled by default and cannot be configured for
indices created with Elasticsearch 6.0 or later, the `include_in_all` setting is
now disallowed for these indices' mappings.
==== Unrecognized `match_mapping_type` options not silently ignored
Previously Elastiscearch would silently ignore any dynamic templates that