honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Default `include_in_all` for numeric-like types to false 

This includes:

- All regular numeric types such as int, long, scaled-float, double, etc
- IP addresses
- Dates
- Geopoints and Geoshapes

Relates to #19784
This commit is contained in:
Lee Hinman 2016-08-19 15:06:10 -06:00
parent cf32f8de34
commit d7e516c0b4
13 changed files with 65 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View File

@ -496,7 +496,10 @@ public class DateFieldMapper extends FieldMapper {
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
// Date fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
String dateAsString;
if (context.externalValueSet()) {
Object dateAsObject = context.externalValue();

5
core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
View File

@ -110,7 +110,10 @@ public class GeoPointFieldMapper extends BaseGeoPointFieldMapper {
}
@Override
protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException {
protected void parse(ParseContext originalContext, GeoPoint point, String geoHash) throws IOException {
// Geopoint fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
if (ignoreMalformed.value() == false) {
if (point.lat() > 90.0 || point.lat() < -90.0) {
throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name());

5
core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View File

@ -430,7 +430,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
}
@Override
public Mapper parse(ParseContext context) throws IOException {
public Mapper parse(ParseContext originalContext) throws IOException {
// Numeric fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
try {
Shape shape = context.parseExternalValue(Shape.class);
if (shape == null) {

5
core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
View File

@ -285,7 +285,10 @@ public class IpFieldMapper extends FieldMapper {
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
// IP fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
Object addressAsObject;
if (context.externalValueSet()) {
addressAsObject = context.externalValue();

5
core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View File

@ -895,7 +895,10 @@ public class NumberFieldMapper extends FieldMapper {
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
// Numeric fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
XContentParser parser = context.parser();
Object value;
Number numericValue = null;

5
core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
View File

@ -364,7 +364,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
// Numeric fields, by default, will not be included in _all
final ParseContext context = originalContext.setIncludeInAllDefault(false);
XContentParser parser = context.parser();
Object value;
Number numericValue = null;

6
core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
View File

@ -178,7 +178,8 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
public void testIncludeInAll() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "date").endObject().endObject()
.startObject("properties").startObject("field").field("type", "date")
.field("include_in_all", true).endObject().endObject()
.endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@ -196,8 +197,7 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
assertEquals("2016-03-11", fields[0].stringValue());
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "date")
.field("include_in_all", false).endObject().endObject()
.startObject("properties").startObject("field").field("type", "date").endObject().endObject()
.endObject().endObject().string();
mapper = parser.parse("type", new CompressedXContent(mapping));

6
core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
View File

@ -186,7 +186,8 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
public void testIncludeInAll() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
.startObject("properties").startObject("field").field("type", "ip")
.field("include_in_all", true).endObject().endObject()
.endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@ -204,8 +205,7 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
assertEquals("::1", fields[0].stringValue());
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "ip")
.field("include_in_all", false).endObject().endObject()
.startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
.endObject().endObject().string();
mapper = parser.parse("type", new CompressedXContent(mapping));

6
core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
View File

@ -264,7 +264,8 @@ public class NumberFieldMapperTests extends ESSingleNodeTestCase {
public void doTestIncludeInAll(String type) throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", type).endObject().endObject()
.startObject("properties").startObject("field").field("type", type)
.field("include_in_all", true).endObject().endObject()
.endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@ -282,8 +283,7 @@ public class NumberFieldMapperTests extends ESSingleNodeTestCase {
assertEquals("123", fields[0].stringValue());
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", type)
.field("include_in_all", false).endObject().endObject()
.startObject("properties").startObject("field").field("type", type).endObject().endObject()
.endObject().endObject().string();
mapper = parser.parse("type", new CompressedXContent(mapping));

6
core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
View File

@ -249,7 +249,8 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
public void testIncludeInAll() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "scaled_float")
.field("scaling_factor", 10.0).endObject().endObject()
.field("scaling_factor", 10.0)
.field("include_in_all", true).endObject().endObject()
.endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@ -268,8 +269,7 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
.field("type", "scaled_float").field("scaling_factor", 10.0)
.field("include_in_all", false).endObject().endObject()
.field("type", "scaled_float").field("scaling_factor", 10.0).endObject().endObject()
.endObject().endObject().string();
mapper = parser.parse("type", new CompressedXContent(mapping));

4
docs/plugins/mapper-attachments.asciidoc
View File

@ -90,10 +90,10 @@ If you get a hit for your indexed document, the plugin should be installed and w
"took": 53,
"hits": {
"total": 1,
"max_score": 0.25811607,
"max_score": 0.28582606,
"hits": [
{
"_score": 0.25811607,
"_score": 0.28582606,
"_index": "trying-out-mapper-attachments",
"_type": "person",
"_id": "1",

29
docs/reference/mapping/fields/all-field.asciidoc
View File

@ -1,14 +1,14 @@
[[mapping-all-field]]
=== `_all` field
The `_all` field is a special _catch-all_ field which concatenates the values
of all of the other fields into one big string, using space as a delimiter, which is then
<<analysis,analyzed>> and indexed, but not stored. This means that it can be
searched, but not retrieved.
The `_all` field is a special _catch-all_ field which concatenates the values of
all of the other string fields into one big string, using space as a delimiter,
which is then <<analysis,analyzed>> and indexed, but not stored. This means that
it can be searched, but not retrieved.
The `_all` field allows you to search for values in documents without knowing
which field contains the value. This makes it a useful option when getting
started with a new dataset. For instance:
The `_all` field allows you to search for string values in documents without
knowing which field contains the value. This makes it a useful option when
getting started with a new dataset. For instance:
[source,js]
--------------------------------
@ -16,7 +16,7 @@ PUT my_index/user/1 <1>
{
"first_name": "John",
"last_name": "Smith",
"date_of_birth": "1970-10-24"
"date_of_birth": "Born 1970-10-24"
}
GET my_index/_search
@ -29,16 +29,17 @@ GET my_index/_search
}
--------------------------------
// CONSOLE
<1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"1970"`, `"10"`, `"24"` ]
<1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"born"`, `"1970"`, `"10"`, `"24"` ]
[NOTE]
.All values treated as strings
.Only string values are added to _all
=============================================================================
The `date_of_birth` field in the above example is recognised as a `date` field
and so will index a single term representing `1970-10-24 00:00:00 UTC`. The
`_all` field, however, treats all values as strings, so the date value is
indexed as the three string terms: `"1970"`, `"24"`, `"10"`.
The `date_of_birth` field in the above example is recognised as a `string` field
and thus will be analyzed as a string, resulting in the terms `"born"`,
`"1970"`, `"24"`, and `"10"`. If the `date_of_birth` field were an actual date
type field, it would not be included in the `_all` field, since `_all` only
contains content from string fields.
It is important to note that the `_all` field combines the original values
from each field as a string. It does not combine the _terms_ from each field.

12
docs/reference/migration/migrate_5_0/mapping.asciidoc
View File

@ -158,6 +158,18 @@ of the 4 bytes used previously. While this will make the index much more
space-efficient, it also means that index time boosts will be less accurately
encoded.
==== `include_in_all` default changed to false for numeric/date/ip/geo types
The default unset value for numeric types for the `include_in_all` mapping
option has been changed from true to false.
This includes:
- All regular numeric types such as int, long, float, scaled-float, double
- IP addresses
- Dates
- Geopoints and Geoshapes
==== `_ttl` and `_timestamp` cannot be created
You can no longer create indexes with `_ttl` or `_timestamp` enabled. Indexes