honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Drop pre 1.4 binary numeric docvalues support

This commit is contained in:
Simon Willnauer 2015-10-08 17:18:53 +02:00
parent 9a662de422
commit fcae618c8c
6 changed files with 4 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View File

@ -79,7 +79,6 @@ public class BinaryFieldMapper extends FieldMapper {
@Override
public BinaryFieldMapper build(BuilderContext context) {
setupFieldType(context);
((BinaryFieldType)fieldType).setTryUncompressing(context.indexCreatedVersion().before(Version.V_2_0_0_beta1));
return new BinaryFieldMapper(name, fieldType, defaultFieldType,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
@ -103,13 +102,11 @@ public class BinaryFieldMapper extends FieldMapper {
}
static final class BinaryFieldType extends MappedFieldType {
private boolean tryUncompressing = false;
public BinaryFieldType() {}
protected BinaryFieldType(BinaryFieldType ref) {
super(ref);
this.tryUncompressing = ref.tryUncompressing;
}
@Override
@ -117,40 +114,12 @@ public class BinaryFieldMapper extends FieldMapper {
return new BinaryFieldType(this);
}
@Override
public boolean equals(Object o) {
if (!super.equals(o)) return false;
BinaryFieldType that = (BinaryFieldType) o;
return Objects.equals(tryUncompressing, that.tryUncompressing);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), tryUncompressing);
}
@Override
public String typeName() {
return CONTENT_TYPE;
}
@Override
public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts, boolean strict) {
super.checkCompatibility(fieldType, conflicts, strict);
BinaryFieldType other = (BinaryFieldType)fieldType;
if (tryUncompressing() != other.tryUncompressing()) {
conflicts.add("mapper [" + names().fullName() + "] has different [try_uncompressing] (IMPOSSIBLE)");
}
}
public boolean tryUncompressing() {
return tryUncompressing;
}
public void setTryUncompressing(boolean tryUncompressing) {
checkIfFrozen();
this.tryUncompressing = tryUncompressing;
}
@Override
public BytesReference value(Object value) {
@ -172,15 +141,7 @@ public class BinaryFieldMapper extends FieldMapper {
throw new ElasticsearchParseException("failed to convert bytes", e);
}
}
try {
if (tryUncompressing) { // backcompat behavior
return CompressorFactory.uncompressIfNeeded(bytes);
} else {
return bytes;
}
} catch (IOException e) {
throw new ElasticsearchParseException("failed to decompress source", e);
}
return bytes;
}
@Override

42
core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.index.mapper.core;
import com.carrotsearch.hppc.DoubleArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
@ -36,8 +34,6 @@ import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -286,17 +282,7 @@ public class DoubleFieldMapper extends NumberFieldMapper {
fields.add(field);
}
if (fieldType().hasDocValues()) {
if (useSortedNumericDocValues) {
addDocValue(context, fields, doubleToSortableLong(value));
} else {
CustomDoubleNumericDocValuesField field = (CustomDoubleNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
if (field != null) {
field.add(value);
} else {
field = new CustomDoubleNumericDocValuesField(fieldType().names().indexName(), value);
context.doc().addWithKey(fieldType().names().indexName(), field);
}
}
addDocValue(context, fields, doubleToSortableLong(value));
}
}
@ -346,30 +332,4 @@ public class DoubleFieldMapper extends NumberFieldMapper {
}
}
public static class CustomDoubleNumericDocValuesField extends CustomNumericDocValuesField {
private final DoubleArrayList values;
public CustomDoubleNumericDocValuesField(String name, double value) {
super(name);
values = new DoubleArrayList();
add(value);
}
public void add(double value) {
values.add(value);
}
@Override
public BytesRef binaryValue() {
CollectionUtils.sortAndDedup(values);
final byte[] bytes = new byte[values.size() * 8];
for (int i = 0; i < values.size(); ++i) {
ByteUtils.writeDoubleLE(values.get(i), bytes, i * 8);
}
return new BytesRef(bytes);
}
}
}

43
core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
View File

@ -19,8 +19,6 @@
package org.elasticsearch.index.mapper.core;
import com.carrotsearch.hppc.FloatArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
@ -37,8 +35,6 @@ import org.elasticsearch.common.Numbers;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -298,17 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper {
fields.add(field);
}
if (fieldType().hasDocValues()) {
if (useSortedNumericDocValues) {
addDocValue(context, fields, floatToSortableInt(value));
} else {
CustomFloatNumericDocValuesField field = (CustomFloatNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
if (field != null) {
field.add(value);
} else {
field = new CustomFloatNumericDocValuesField(fieldType().names().indexName(), value);
context.doc().addWithKey(fieldType().names().indexName(), field);
}
}
addDocValue(context, fields, floatToSortableInt(value));
}
}
@ -357,31 +343,4 @@ public class FloatFieldMapper extends NumberFieldMapper {
return Float.toString(number);
}
}
public static class CustomFloatNumericDocValuesField extends CustomNumericDocValuesField {
private final FloatArrayList values;
public CustomFloatNumericDocValuesField(String name, float value) {
super(name);
values = new FloatArrayList();
add(value);
}
public void add(float value) {
values.add(value);
}
@Override
public BytesRef binaryValue() {
CollectionUtils.sortAndDedup(values);
final byte[] bytes = new byte[values.size() * 4];
for (int i = 0; i < values.size(); ++i) {
ByteUtils.writeFloatLE(values.get(i), bytes, i * 4);
}
return new BytesRef(bytes);
}
}
}

60
core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper.core;
import com.carrotsearch.hppc.LongArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
@ -31,14 +30,10 @@ import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.*;
@ -170,21 +165,12 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
protected Explicit<Boolean> coerce;
/**
* True if index version is 1.4+
* <p>
* In this case numerics are encoded with SORTED_NUMERIC docvalues,
* otherwise for older indexes we must continue to write BINARY (for now)
*/
protected final boolean useSortedNumericDocValues;
protected NumberFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.ignoreMalformed = ignoreMalformed;
this.coerce = coerce;
this.useSortedNumericDocValues = Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1);
}
@Override
@ -225,17 +211,7 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
protected abstract void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException;
protected final void addDocValue(ParseContext context, List<Field> fields, long value) {
if (useSortedNumericDocValues) {
fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value));
} else {
CustomLongNumericDocValuesField field = (CustomLongNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName());
if (field != null) {
field.add(value);
} else {
field = new CustomLongNumericDocValuesField(fieldType().names().indexName(), value);
context.doc().addWithKey(fieldType().names().indexName(), field);
}
}
fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value));
}
/**
@ -414,40 +390,6 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
}
public static class CustomLongNumericDocValuesField extends CustomNumericDocValuesField {
private final LongArrayList values;
public CustomLongNumericDocValuesField(String name, long value) {
super(name);
values = new LongArrayList();
add(value);
}
public void add(long value) {
values.add(value);
}
@Override
public BytesRef binaryValue() {
CollectionUtils.sortAndDedup(values);
// here is the trick:
// - the first value is zig-zag encoded so that eg. -5 would become positive and would be better compressed by vLong
// - for other values, we only encode deltas using vLong
final byte[] bytes = new byte[values.size() * ByteUtils.MAX_BYTES_VLONG];
final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
ByteUtils.writeVLong(out, ByteUtils.zigZagEncode(values.get(0)));
for (int i = 1; i < values.size(); ++i) {
final long delta = values.get(i) - values.get(i - 1);
ByteUtils.writeVLong(out, delta);
}
return new BytesRef(bytes, 0, out.getPosition());
}
}
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);

1
core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
View File

@ -97,7 +97,6 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
if (v.snapshot()) continue;
if (v.onOrBefore(Version.V_2_0_0_beta1)) continue;
if (v.equals(Version.CURRENT)) continue;
expectedVersions.add(v.toString());
}
}

11
core/src/test/java/org/elasticsearch/index/mapper/core/BinaryFieldTypeTests.java
View File

@ -28,15 +28,4 @@ public class BinaryFieldTypeTests extends FieldTypeTestCase {
protected MappedFieldType createDefaultFieldType() {
return new BinaryFieldMapper.BinaryFieldType();
}
@Before
public void setupProperties() {
addModifier(new Modifier("try_uncompressing", false, true) {
@Override
public void modify(MappedFieldType ft) {
BinaryFieldMapper.BinaryFieldType bft = (BinaryFieldMapper.BinaryFieldType)ft;
bft.setTryUncompressing(!bft.tryUncompressing());
}
});
}
}