honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow big integers and decimals to be mapped dynamically. (#42827) 

This PR proposes to model big integers as longs (and big decimals as doubles)
in the context of dynamic mappings.

Previously, the dynamic mapping logic did not recognize big integers or
decimals, and would an error of the form "No matching token for number_type
[BIG_INTEGER]" when a dynamic big integer was encountered. It now accepts these
numeric types and interprets them as 'long' and 'double' respectively. This
allows `dynamic_templates` to accept and and remap them as another type such as
`keyword` or `scaled_float`.

Addresses #37846.
This commit is contained in:
Julie Tibshirani 2019-06-13 12:07:40 -07:00
parent 3928c624a3
commit 4b1d8e4433
4 changed files with 71 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
View File

@ -114,7 +114,7 @@ public interface XContentParser extends Closeable {
}
enum NumberType {
INT, LONG, FLOAT, DOUBLE
INT, BIG_INTEGER, LONG, FLOAT, DOUBLE, BIG_DECIMAL
}
XContentType contentType();

4
libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
View File

@ -199,12 +199,16 @@ public class JsonXContentParser extends AbstractXContentParser {
switch (numberType) {
case INT:
return NumberType.INT;
case BIG_INTEGER:
return NumberType.BIG_INTEGER;
case LONG:
return NumberType.LONG;
case FLOAT:
return NumberType.FLOAT;
case DOUBLE:
return NumberType.DOUBLE;
case BIG_DECIMAL:
return NumberType.BIG_DECIMAL;
}
throw new IllegalStateException("No matching token for number_type [" + numberType + "]");
}

8
server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -765,13 +765,17 @@ final class DocumentParser {
return builder;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) {
if (numberType == XContentParser.NumberType.INT
|| numberType == XContentParser.NumberType.LONG
|| numberType == XContentParser.NumberType.BIG_INTEGER) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
if (builder == null) {
builder = newLongBuilder(currentFieldName, context.indexSettings().getIndexVersionCreated());
}
return builder;
} else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE) {
} else if (numberType == XContentParser.NumberType.FLOAT
|| numberType == XContentParser.NumberType.DOUBLE
|| numberType == XContentParser.NumberType.BIG_DECIMAL) {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
if (builder == null) {
// no templates are defined, we use float by default instead of double

60
server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
@ -37,6 +39,8 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
@ -706,6 +710,62 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
assertEquals(0, doc.rootDoc().getFields("foo").length);
}
public void testDynamicBigInteger() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startArray("dynamic_templates").startObject()
.startObject("big-integer-to-keyword")
.field("match", "big-*")
.field("match_mapping_type", "long")
.startObject("mapping").field("type", "keyword").endObject()
.endObject()
.endObject().endArray()
.endObject()
.endObject());
DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
BigInteger value = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE);
BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("big-integer", value)
.endObject());
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON));
IndexableField[] fields = doc.rootDoc().getFields("big-integer");
assertEquals(2, fields.length);
assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType);
assertEquals(new BytesRef(value.toString()), fields[0].binaryValue());
}
public void testDynamicBigDecimal() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startArray("dynamic_templates").startObject()
.startObject("big-decimal-to-scaled-float")
.field("match", "big-*")
.field("match_mapping_type", "double")
.startObject("mapping")
.field("type", "keyword")
.endObject()
.endObject()
.endObject().endArray()
.endObject()
.endObject());
BigDecimal value = BigDecimal.valueOf(Double.MAX_VALUE).add(BigDecimal.valueOf(10.1));
DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("big-decimal", value)
.endObject());
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON));
IndexableField[] fields = doc.rootDoc().getFields("big-decimal");
assertEquals(2, fields.length);
assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType);
assertEquals(new BytesRef(value.toString()), fields[0].binaryValue());
}
public void testDynamicDottedFieldNameLongArray() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")