honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed how the stored values of numeric fields are stored in the index. Before numeric values were stored in binary representation, now the values in numeric representation.

This commit is contained in:
Martijn van Groningen 2013-01-03 18:22:17 +01:00
parent 0032b334c5
commit 7cf80aca99
9 changed files with 131 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
View File

@ -364,7 +364,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
private final NumberFieldMapper mapper;
public CustomByteNumericField(NumberFieldMapper mapper, byte number, FieldType fieldType) {
super(mapper, mapper.fieldType.stored() ? new byte[]{number} : null, fieldType);
super(mapper, mapper.fieldType.stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

2
src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -364,7 +364,7 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
private final NumberFieldMapper mapper;
public CustomDoubleNumericField(NumberFieldMapper mapper, double number, FieldType fieldType) {
super(mapper, mapper.fieldType().stored() ? Numbers.doubleToBytes(number) : null, fieldType);
super(mapper, mapper.fieldType().stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

2
src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -359,7 +359,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
private final NumberFieldMapper mapper;
public CustomFloatNumericField(NumberFieldMapper mapper, float number, FieldType fieldType) {
super(mapper, mapper.fieldType().stored() ? Numbers.floatToBytes(number) : null, fieldType);
super(mapper, mapper.fieldType().stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

2
src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
View File

@ -364,7 +364,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
private final NumberFieldMapper mapper;
public CustomIntegerNumericField(NumberFieldMapper mapper, int number, FieldType fieldType) {
super(mapper, mapper.fieldType().stored() ? Numbers.intToBytes(number) : null, fieldType);
super(mapper, mapper.fieldType().stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

2
src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
View File

@ -363,7 +363,7 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
private final NumberFieldMapper mapper;
public CustomLongNumericField(NumberFieldMapper mapper, long number, FieldType fieldType) {
super(mapper, mapper.fieldType.stored() ? Numbers.longToBytes(number) : null, fieldType);
super(mapper, mapper.fieldType.stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

5
src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -25,7 +25,6 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
@ -290,11 +289,11 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
protected final NumberFieldMapper mapper;
public CustomNumericField(NumberFieldMapper mapper, byte[] value, FieldType fieldType) {
public CustomNumericField(NumberFieldMapper mapper, Number value, FieldType fieldType) {
super(mapper.names().indexName(), fieldType);
this.mapper = mapper;
if (value != null) {
this.fieldsData = new BytesRef(value);
this.fieldsData = value;
}
}

2
src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
View File

@ -367,7 +367,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
private final NumberFieldMapper mapper;
public CustomShortNumericField(NumberFieldMapper mapper, short number, FieldType fieldType) {
super(mapper, mapper.fieldType().stored() ? Numbers.shortToBytes(number) : null, fieldType);
super(mapper, mapper.fieldType().stored() ? number : null, fieldType);
this.mapper = mapper;
this.number = number;
}

42
src/test/java/org/elasticsearch/test/unit/index/mapper/geo/LatLonMappingGeoPointTests.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.test.unit.index.mapper.geo;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
@ -187,9 +185,9 @@ public class LatLonMappingGeoPointTests {
.bytes());
assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
assertThat(doc.rootDoc().getField("point.lat").binaryValue(), nullValue());
assertThat(doc.rootDoc().getField("point.lat").numericValue(), nullValue());
assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
assertThat(doc.rootDoc().getField("point.lon").binaryValue(), nullValue());
assertThat(doc.rootDoc().getField("point.lon").numericValue(), nullValue());
assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}
@ -209,9 +207,9 @@ public class LatLonMappingGeoPointTests {
.bytes());
assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lat").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lon").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}
@ -235,11 +233,11 @@ public class LatLonMappingGeoPointTests {
assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.4)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.5)));
assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
}
@ -277,9 +275,9 @@ public class LatLonMappingGeoPointTests {
.bytes());
assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lat").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lon").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}
@ -302,11 +300,11 @@ public class LatLonMappingGeoPointTests {
assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.4)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.5)));
assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
}
@ -365,9 +363,9 @@ public class LatLonMappingGeoPointTests {
.bytes());
assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lat").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(doc.rootDoc().getField("point.lat").numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lon").binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getField("point.lon").numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}
@ -390,11 +388,11 @@ public class LatLonMappingGeoPointTests {
assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.2)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[0].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.3)));
assertThat(doc.rootDoc().getFields("point.lat")[0].numericValue().doubleValue(), equalTo(1.2));
assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.4)));
assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[1].binaryValue()).bytes, equalTo(Numbers.doubleToBytes(1.5)));
assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
}
}

103
src/test/java/org/elasticsearch/test/unit/index/mapper/lucene/StoredNumericValuesTest.java Normal file
View File

@ -0,0 +1,103 @@
package org.elasticsearch.test.unit.index.mapper.lucene;
import org.apache.lucene.document.*;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
import java.util.*;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@Test
public class StoredNumericValuesTest {
@Test
public void testBytesAndNumericRepresentation() throws Exception {
IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
String mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field1").field("type", "integer").field("store", "yes").endObject()
.startObject("field2").field("type", "float").field("store", "yes").endObject()
.startObject("field3").field("type", "long").field("store", "yes").endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = mapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field1", 1)
.field("field2", 1.1)
.startArray("field3").value(1).value(2).value(3).endArray()
.endObject()
.bytes());
writer.addDocument(doc.rootDoc(), doc.analyzer());
// Indexing a doc in the old way
FieldType fieldType = new FieldType();
fieldType.setStored(true);
fieldType.setNumericType(FieldType.NumericType.INT);
Document doc2 = new Document();
doc2.add(new StoredField("field1", new BytesRef(Numbers.intToBytes(1))));
doc2.add(new StoredField("field2", new BytesRef(Numbers.floatToBytes(1.1f))));
doc2.add(new StoredField("field3", new BytesRef(Numbers.longToBytes(1l))));
doc2.add(new StoredField("field3", new BytesRef(Numbers.longToBytes(2l))));
doc2.add(new StoredField("field3", new BytesRef(Numbers.longToBytes(3l))));
writer.addDocument(doc2);
DirectoryReader reader = DirectoryReader.open(writer, true);
IndexSearcher searcher = new IndexSearcher(reader);
Set<String> fields = new HashSet<String>(Arrays.asList("field1", "field2", "field3"));
CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fields, false);
searcher.doc(0, fieldsVisitor);
fieldsVisitor.postProcess(mapper);
assertThat(fieldsVisitor.fields().size(), equalTo(3));
assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));
assertThat((Integer) fieldsVisitor.fields().get("field1").get(0), equalTo(1));
assertThat(fieldsVisitor.fields().get("field2").size(), equalTo(1));
assertThat((Float) fieldsVisitor.fields().get("field2").get(0), equalTo(1.1f));
assertThat(fieldsVisitor.fields().get("field3").size(), equalTo(3));
assertThat((Long) fieldsVisitor.fields().get("field3").get(0), equalTo(1l));
assertThat((Long) fieldsVisitor.fields().get("field3").get(1), equalTo(2l));
assertThat((Long) fieldsVisitor.fields().get("field3").get(2), equalTo(3l));
// Make sure the doc gets loaded as if it was stored in the new way
fieldsVisitor.reset();
searcher.doc(1, fieldsVisitor);
fieldsVisitor.postProcess(mapper);
assertThat(fieldsVisitor.fields().size(), equalTo(3));
assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));
assertThat((Integer) fieldsVisitor.fields().get("field1").get(0), equalTo(1));
assertThat(fieldsVisitor.fields().get("field2").size(), equalTo(1));
assertThat((Float) fieldsVisitor.fields().get("field2").get(0), equalTo(1.1f));
assertThat(fieldsVisitor.fields().get("field3").size(), equalTo(3));
assertThat((Long) fieldsVisitor.fields().get("field3").get(0), equalTo(1l));
assertThat((Long) fieldsVisitor.fields().get("field3").get(1), equalTo(2l));
assertThat((Long) fieldsVisitor.fields().get("field3").get(2), equalTo(3l));
reader.close();
writer.close();
}
}