From fcae618c8ca5a2f0d1e785910eb9bafe7ce05150 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 8 Oct 2015 17:18:53 +0200 Subject: [PATCH] Drop pre 1.4 binary numeric docvalues support --- .../index/mapper/core/BinaryFieldMapper.java | 41 +------------ .../index/mapper/core/DoubleFieldMapper.java | 42 +------------ .../index/mapper/core/FloatFieldMapper.java | 43 +------------ .../index/mapper/core/NumberFieldMapper.java | 60 +------------------ .../bwcompat/RestoreBackwardsCompatIT.java | 1 - .../mapper/core/BinaryFieldTypeTests.java | 11 ---- 6 files changed, 4 insertions(+), 194 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java index 78d038526b3..7468f4fb2f6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java @@ -79,7 +79,6 @@ public class BinaryFieldMapper extends FieldMapper { @Override public BinaryFieldMapper build(BuilderContext context) { setupFieldType(context); - ((BinaryFieldType)fieldType).setTryUncompressing(context.indexCreatedVersion().before(Version.V_2_0_0_beta1)); return new BinaryFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } @@ -103,13 +102,11 @@ public class BinaryFieldMapper extends FieldMapper { } static final class BinaryFieldType extends MappedFieldType { - private boolean tryUncompressing = false; public BinaryFieldType() {} protected BinaryFieldType(BinaryFieldType ref) { super(ref); - this.tryUncompressing = ref.tryUncompressing; } @Override @@ -117,40 +114,12 @@ public class BinaryFieldMapper extends FieldMapper { return new BinaryFieldType(this); } - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - BinaryFieldType that = (BinaryFieldType) o; - return Objects.equals(tryUncompressing, that.tryUncompressing); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), tryUncompressing); - } @Override public String typeName() { return CONTENT_TYPE; } - @Override - public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { - super.checkCompatibility(fieldType, conflicts, strict); - BinaryFieldType other = (BinaryFieldType)fieldType; - if (tryUncompressing() != other.tryUncompressing()) { - conflicts.add("mapper [" + names().fullName() + "] has different [try_uncompressing] (IMPOSSIBLE)"); - } - } - - public boolean tryUncompressing() { - return tryUncompressing; - } - - public void setTryUncompressing(boolean tryUncompressing) { - checkIfFrozen(); - this.tryUncompressing = tryUncompressing; - } @Override public BytesReference value(Object value) { @@ -172,15 +141,7 @@ public class BinaryFieldMapper extends FieldMapper { throw new ElasticsearchParseException("failed to convert bytes", e); } } - try { - if (tryUncompressing) { // backcompat behavior - return CompressorFactory.uncompressIfNeeded(bytes); - } else { - return bytes; - } - } catch (IOException e) { - throw new ElasticsearchParseException("failed to decompress source", e); - } + return bytes; } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 7f06c223e62..0e512bf4281 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -19,8 +19,6 @@ package org.elasticsearch.index.mapper.core; -import com.carrotsearch.hppc.DoubleArrayList; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; @@ -36,8 +34,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -286,17 +282,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { fields.add(field); } if (fieldType().hasDocValues()) { - if (useSortedNumericDocValues) { - addDocValue(context, fields, doubleToSortableLong(value)); - } else { - CustomDoubleNumericDocValuesField field = (CustomDoubleNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName()); - if (field != null) { - field.add(value); - } else { - field = new CustomDoubleNumericDocValuesField(fieldType().names().indexName(), value); - context.doc().addWithKey(fieldType().names().indexName(), field); - } - } + addDocValue(context, fields, doubleToSortableLong(value)); } } @@ -346,30 +332,4 @@ public class DoubleFieldMapper extends NumberFieldMapper { } } - public static class CustomDoubleNumericDocValuesField extends CustomNumericDocValuesField { - - private final DoubleArrayList values; - - public CustomDoubleNumericDocValuesField(String name, double value) { - super(name); - values = new DoubleArrayList(); - add(value); - } - - public void add(double value) { - values.add(value); - } - - @Override - public BytesRef binaryValue() { - CollectionUtils.sortAndDedup(values); - - final byte[] bytes = new byte[values.size() * 8]; - for (int i = 0; i < values.size(); ++i) { - ByteUtils.writeDoubleLE(values.get(i), bytes, i * 8); - } - return new BytesRef(bytes); - } - - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index caeb2d7a188..9a607ffd415 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -19,8 +19,6 @@ package org.elasticsearch.index.mapper.core; -import com.carrotsearch.hppc.FloatArrayList; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; @@ -37,8 +35,6 @@ import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -298,17 +294,7 @@ public class FloatFieldMapper extends NumberFieldMapper { fields.add(field); } if (fieldType().hasDocValues()) { - if (useSortedNumericDocValues) { - addDocValue(context, fields, floatToSortableInt(value)); - } else { - CustomFloatNumericDocValuesField field = (CustomFloatNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName()); - if (field != null) { - field.add(value); - } else { - field = new CustomFloatNumericDocValuesField(fieldType().names().indexName(), value); - context.doc().addWithKey(fieldType().names().indexName(), field); - } - } + addDocValue(context, fields, floatToSortableInt(value)); } } @@ -357,31 +343,4 @@ public class FloatFieldMapper extends NumberFieldMapper { return Float.toString(number); } } - - public static class CustomFloatNumericDocValuesField extends CustomNumericDocValuesField { - - private final FloatArrayList values; - - public CustomFloatNumericDocValuesField(String name, float value) { - super(name); - values = new FloatArrayList(); - add(value); - } - - public void add(float value) { - values.add(value); - } - - @Override - public BytesRef binaryValue() { - CollectionUtils.sortAndDedup(values); - - final byte[] bytes = new byte[values.size() * 4]; - for (int i = 0; i < values.size(); ++i) { - ByteUtils.writeFloatLE(values.get(i), bytes, i * 4); - } - return new BytesRef(bytes); - } - - } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index 78406c2afbc..3fba511fb52 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper.core; -import com.carrotsearch.hppc.LongArrayList; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.NumericTokenStream; import org.apache.lucene.analysis.TokenStream; @@ -31,14 +30,10 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.search.Query; -import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.*; @@ -170,21 +165,12 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM protected Explicit coerce; - /** - * True if index version is 1.4+ - *

- * In this case numerics are encoded with SORTED_NUMERIC docvalues, - * otherwise for older indexes we must continue to write BINARY (for now) - */ - protected final boolean useSortedNumericDocValues; - protected NumberFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; this.coerce = coerce; - this.useSortedNumericDocValues = Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1); } @Override @@ -225,17 +211,7 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM protected abstract void innerParseCreateField(ParseContext context, List fields) throws IOException; protected final void addDocValue(ParseContext context, List fields, long value) { - if (useSortedNumericDocValues) { - fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value)); - } else { - CustomLongNumericDocValuesField field = (CustomLongNumericDocValuesField) context.doc().getByKey(fieldType().names().indexName()); - if (field != null) { - field.add(value); - } else { - field = new CustomLongNumericDocValuesField(fieldType().names().indexName(), value); - context.doc().addWithKey(fieldType().names().indexName(), field); - } - } + fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value)); } /** @@ -414,40 +390,6 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM } - - public static class CustomLongNumericDocValuesField extends CustomNumericDocValuesField { - - private final LongArrayList values; - - public CustomLongNumericDocValuesField(String name, long value) { - super(name); - values = new LongArrayList(); - add(value); - } - - public void add(long value) { - values.add(value); - } - - @Override - public BytesRef binaryValue() { - CollectionUtils.sortAndDedup(values); - - // here is the trick: - // - the first value is zig-zag encoded so that eg. -5 would become positive and would be better compressed by vLong - // - for other values, we only encode deltas using vLong - final byte[] bytes = new byte[values.size() * ByteUtils.MAX_BYTES_VLONG]; - final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - ByteUtils.writeVLong(out, ByteUtils.zigZagEncode(values.get(0))); - for (int i = 1; i < values.size(); ++i) { - final long delta = values.get(i) - values.get(i - 1); - ByteUtils.writeVLong(out, delta); - } - return new BytesRef(bytes, 0, out.getPosition()); - } - - } - @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index c88261d73d6..740b185e745 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -97,7 +97,6 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { if (v.snapshot()) continue; if (v.onOrBefore(Version.V_2_0_0_beta1)) continue; if (v.equals(Version.CURRENT)) continue; - expectedVersions.add(v.toString()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/BinaryFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/BinaryFieldTypeTests.java index f241d555e12..7ab78864fb7 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/BinaryFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/BinaryFieldTypeTests.java @@ -28,15 +28,4 @@ public class BinaryFieldTypeTests extends FieldTypeTestCase { protected MappedFieldType createDefaultFieldType() { return new BinaryFieldMapper.BinaryFieldType(); } - - @Before - public void setupProperties() { - addModifier(new Modifier("try_uncompressing", false, true) { - @Override - public void modify(MappedFieldType ft) { - BinaryFieldMapper.BinaryFieldType bft = (BinaryFieldMapper.BinaryFieldType)ft; - bft.setTryUncompressing(!bft.tryUncompressing()); - } - }); - } }