From f2d8a3588850494a59f4da98cecea281e504e773 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 8 Dec 2015 18:05:36 +0100 Subject: [PATCH] Dynamically map floating-point numbers as floats instead of doubles. Close #13851 --- .../index/mapper/DocumentParser.java | 10 +++++-- .../elasticsearch/index/mapper/Mapper.java | 2 +- .../fieldstats/FieldStatsTests.java | 16 +++++----- .../index/mapper/DynamicMappingTests.java | 29 +++++++++++++++++++ docs/reference/migration/migrate_3_0.asciidoc | 7 +++++ 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e5e3387950e..ce2cbd4e931 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -584,7 +584,10 @@ class DocumentParser implements Closeable { if (context.parser().estimatedNumberType()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { - builder = MapperBuilders.doubleField(currentFieldName); + // no templates are defined, we use float by default instead of double + // since this is much more space-efficient and should be enough most of + // the time + builder = MapperBuilders.floatField(currentFieldName); } return builder; } else { @@ -597,7 +600,10 @@ class DocumentParser implements Closeable { } else if (numberType == XContentParser.NumberType.DOUBLE) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { - builder = MapperBuilders.doubleField(currentFieldName); + // no templates are defined, we use float by default instead of double + // since this is much more space-efficient and should be enough most of + // the time + builder = MapperBuilders.floatField(currentFieldName); } return builder; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java index c9877410c30..33a4dabd3be 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -148,7 +148,7 @@ public abstract class Mapper implements ToXContent, Iterable { }; } - class MultiFieldParserContext extends ParserContext { + static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { super(in.type(), in.analysisService, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher()); } diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java index aec73f245cd..e25b95be578 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java @@ -67,7 +67,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { } public void testString() { - createIndex("test", Settings.EMPTY, "field", "value", "type=string"); + createIndex("test", Settings.EMPTY, "test", "field", "type=string"); for (int value = 0; value <= 10; value++) { client().prepareIndex("test", "test").setSource("field", String.format(Locale.ENGLISH, "%03d", value)).get(); } @@ -85,7 +85,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { public void testDouble() { String fieldName = "field"; - createIndex("test", Settings.EMPTY, fieldName, "value", "type=double"); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=double"); for (double value = -1; value <= 9; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -102,7 +102,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase { public void testFloat() { String fieldName = "field"; - createIndex("test", Settings.EMPTY, fieldName, "value", "type=float"); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=float"); for (float value = -1; value <= 9; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -112,14 +112,14 @@ public class FieldStatsTests extends ESSingleNodeTestCase { assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11l)); assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100)); - assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1.0)); - assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9.0)); + assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1f)); + assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9f)); assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1))); assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9))); } private void testNumberRange(String fieldName, String fieldType, long min, long max) { - createIndex("test", Settings.EMPTY, fieldName, "value", "type=" + fieldType); + createIndex("test", Settings.EMPTY, "test", fieldName, "type=" + fieldType); for (long value = min; value <= max; value++) { client().prepareIndex("test", "test").setSource(fieldName, value).get(); } @@ -180,11 +180,11 @@ public class FieldStatsTests extends ESSingleNodeTestCase { } public void testInvalidField() { - createIndex("test1", Settings.EMPTY, "field1", "value", "type=string"); + createIndex("test1", Settings.EMPTY, "test", "field1", "type=string"); client().prepareIndex("test1", "test").setSource("field1", "a").get(); client().prepareIndex("test1", "test").setSource("field1", "b").get(); - createIndex("test2", Settings.EMPTY, "field2", "value", "type=string"); + createIndex("test2", Settings.EMPTY, "test", "field2", "type=string"); client().prepareIndex("test2", "test").setSource("field2", "a").get(); client().prepareIndex("test2", "test").setSource("field2", "b").get(); client().admin().indices().prepareRefresh().get(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index f01df630ea7..966ea01e95c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -28,15 +29,21 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper; +import org.elasticsearch.index.mapper.core.FloatFieldMapper; import org.elasticsearch.index.mapper.core.IntegerFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; public class DynamicMappingTests extends ESSingleNodeTestCase { @@ -407,4 +414,26 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { // expected } } + + public void testDefaultFloatingPointMappings() throws IOException { + DocumentMapper mapper = createIndex("test").mapperService().documentMapperWithAutoCreate("type").getDocumentMapper(); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder()); + doTestDefaultFloatingPointMappings(mapper, XContentFactory.cborBuilder()); + } + + private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentBuilder builder) throws IOException { + BytesReference source = builder.startObject() + .field("foo", 3.2f) // float + .field("bar", 3.2d) // double + .field("baz", (double) 3.2f) // double that can be accurately represented as a float + .endObject().bytes(); + ParsedDocument parsedDocument = mapper.parse("index", "type", "id", source); + Mapping update = parsedDocument.dynamicMappingsUpdate(); + assertNotNull(update); + assertThat(update.root().getMapper("foo"), instanceOf(FloatFieldMapper.class)); + assertThat(update.root().getMapper("bar"), instanceOf(FloatFieldMapper.class)); + assertThat(update.root().getMapper("baz"), instanceOf(FloatFieldMapper.class)); + } } diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index b8683bc6fd0..112b50872fc 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -206,6 +206,13 @@ cluster settings please use the settings update API and set their superseded key The `transform` feature from mappings has been removed. It made issues very hard to debug. +==== Default number mappings + +When a floating-point number is encountered, it is now dynamically mapped as a +float by default instead of a double. The reasoning is that floats should be +more than enough for most cases but would decrease storage requirements +significantly. + [[breaking_30_plugins]] === Plugin changes