From f1f3c241fd5dd03a326cfb595c7b04045cd9f181 Mon Sep 17 00:00:00 2001 From: uboness Date: Sun, 20 Jan 2013 22:40:31 +0100 Subject: [PATCH] short field data implementation --- .../fielddata/IndexFieldDataService.java | 2 + .../index/fielddata/StringValues.java | 99 +- .../plain/FloatArrayAtomicFieldData.java | 2 +- .../plain/ShortArrayAtomicFieldData.java | 928 ++++++++++++++++++ .../plain/ShortArrayIndexFieldData.java | 157 +++ .../index/mapper/core/ShortFieldMapper.java | 2 +- .../index/fielddata/ShortFieldDataTests.java | 37 + 7 files changed, 1221 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 2032e59a725..793f46da9ec 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -47,6 +47,7 @@ public class IndexFieldDataService extends AbstractIndexComponent { .put("string", new ConcreteBytesRefIndexFieldData.Builder()) .put("float", new FloatArrayIndexFieldData.Builder()) .put("double", new DoubleArrayIndexFieldData.Builder()) + .put("short", new ShortArrayIndexFieldData.Builder()) .put("int", new IntArrayIndexFieldData.Builder()) .put("long", new LongArrayIndexFieldData.Builder()) .put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder()) @@ -56,6 +57,7 @@ public class IndexFieldDataService extends AbstractIndexComponent { .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder()) .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder()) .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder()) + .put(Tuple.tuple("short", "array"), new ShortArrayIndexFieldData.Builder()) .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder()) .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder()) .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder()) diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java index 3178e25665a..577cd6f02b4 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java +++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java @@ -20,10 +20,7 @@ package org.elasticsearch.index.fielddata; import org.elasticsearch.ElasticSearchIllegalStateException; -import org.elasticsearch.index.fielddata.util.FloatArrayRef; -import org.elasticsearch.index.fielddata.util.IntArrayRef; -import org.elasticsearch.index.fielddata.util.LongArrayRef; -import org.elasticsearch.index.fielddata.util.StringArrayRef; +import org.elasticsearch.index.fielddata.util.*; /** */ @@ -103,6 +100,100 @@ public interface StringValues { } } + public static class ShortBased implements StringValues { + + private final ShortValues values; + + private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1); + private final ValuesIter valuesIter = new ValuesIter(); + private final Proc proc = new Proc(); + + public ShortBased(ShortValues values) { + this.values = values; + } + + @Override + public boolean isMultiValued() { + return values.isMultiValued(); + } + + @Override + public boolean hasValue(int docId) { + return values.hasValue(docId); + } + + @Override + public String getValue(int docId) { + if (!values.hasValue(docId)) { + return null; + } + return Short.toString(values.getValue(docId)); + } + + @Override + public StringArrayRef getValues(int docId) { + ShortArrayRef arrayRef = values.getValues(docId); + int size = arrayRef.size(); + if (size == 0) return StringArrayRef.EMPTY; + + arrayScratch.reset(size); + for (int i = arrayRef.start; i < arrayRef.end; i++) { + arrayScratch.values[arrayScratch.end++] = Short.toString(arrayRef.values[i]); + } + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return valuesIter.reset(values.getIter(docId)); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + values.forEachValueInDoc(docId, this.proc.reset(proc)); + } + + static class ValuesIter implements Iter { + + private ShortValues.Iter iter; + + private ValuesIter reset(ShortValues.Iter iter) { + this.iter = iter; + return this; + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public String next() { + return Short.toString(iter.next()); + } + } + + static class Proc implements ShortValues.ValueInDocProc { + + private ValueInDocProc proc; + + private Proc reset(ValueInDocProc proc) { + this.proc = proc; + return this; + } + + @Override + public void onValue(int docId, short value) { + proc.onValue(docId, Short.toString(value)); + } + + @Override + public void onMissing(int docId) { + proc.onMissing(docId); + } + } + } + public static class IntBased implements StringValues { private final IntValues values; diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java index 765a967ecdc..afba43c9f3a 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java @@ -416,7 +416,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat public double next() { float value = values[ord]; ord = ordsIter.next(); - return value; + return (double) value; } } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java new file mode 100644 index 00000000000..236bb4d89b0 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java @@ -0,0 +1,928 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.common.RamUsage; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.ordinals.Ordinals; +import org.elasticsearch.index.fielddata.util.DoubleArrayRef; +import org.elasticsearch.index.fielddata.util.IntArrayRef; +import org.elasticsearch.index.fielddata.util.LongArrayRef; +import org.elasticsearch.index.fielddata.util.ShortArrayRef; + +/** + */ +public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldData { + + protected final short[] values; + private final int numDocs; + + protected long size = -1; + + public ShortArrayAtomicFieldData(short[] values, int numDocs) { + this.values = values; + this.numDocs = numDocs; + } + + @Override + public int getNumDocs() { + return numDocs; + } + + public static class WithOrdinals extends ShortArrayAtomicFieldData { + + private final Ordinals ordinals; + + public WithOrdinals(short[] values, int numDocs, Ordinals ordinals) { + super(values, numDocs); + this.ordinals = ordinals; + } + + @Override + public boolean isMultiValued() { + return ordinals.isMultiValued(); + } + + @Override + public boolean isValuesOrdered() { + return true; + } + + @Override + public long getMemorySizeInBytes() { + if (size == -1) { + size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT) + ordinals.getMemorySizeInBytes(); + } + return size; + } + + @Override + public BytesValues getBytesValues() { + return new BytesValues.StringBased(getStringValues()); + } + + @Override + public HashedBytesValues getHashedBytesValues() { + return new HashedBytesValues.StringBased(getStringValues()); + } + + @Override + public StringValues getStringValues() { + return new StringValues.ShortBased(getShortValues()); + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.NumericDouble(getDoubleValues()); + } + + @Override + public ByteValues getByteValues() { + return new ByteValues.LongBased(getLongValues()); + } + + @Override + public ShortValues getShortValues() { + return new ShortValues(values, ordinals.ordinals()); + } + + @Override + public IntValues getIntValues() { + return new IntValues.LongBased(getLongValues()); + } + + @Override + public LongValues getLongValues() { + return new LongValues(values, ordinals.ordinals()); + } + + @Override + public FloatValues getFloatValues() { + return new FloatValues.DoubleBased(getDoubleValues()); + } + + @Override + public DoubleValues getDoubleValues() { + return new DoubleValues(values, ordinals.ordinals()); + } + + static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues { + + private final short[] values; + private final Ordinals.Docs ordinals; + + private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1); + private final ValuesIter iter; + + ShortValues(short[] values, Ordinals.Docs ordinals) { + this.values = values; + this.ordinals = ordinals; + this.iter = new ValuesIter(values); + } + + @Override + public boolean isMultiValued() { + return ordinals.isMultiValued(); + } + + @Override + public boolean hasValue(int docId) { + return ordinals.getOrd(docId) != 0; + } + + @Override + public short getValue(int docId) { + return values[ordinals.getOrd(docId)]; + } + + @Override + public short getValueMissing(int docId, short missingValue) { + int ord = ordinals.getOrd(docId); + if (ord == 0) { + return missingValue; + } else { + return values[ord]; + } + } + + @Override + public ShortArrayRef getValues(int docId) { + IntArrayRef ords = ordinals.getOrds(docId); + int size = ords.size(); + if (size == 0) return ShortArrayRef.EMPTY; + + arrayScratch.reset(size); + for (int i = ords.start; i < ords.end; i++) { + arrayScratch.values[arrayScratch.end++] = values[ords.values[i]]; + } + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset(ordinals.getIter(docId)); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + Ordinals.Docs.Iter iter = ordinals.getIter(docId); + int ord = iter.next(); + if (ord == 0) { + proc.onMissing(docId); + return; + } + do { + proc.onValue(docId, values[ord]); + } while ((ord = iter.next()) != 0); + } + + static class ValuesIter implements Iter { + + private final short[] values; + private Ordinals.Docs.Iter ordsIter; + private int ord; + + ValuesIter(short[] values) { + this.values = values; + } + + public ValuesIter reset(Ordinals.Docs.Iter ordsIter) { + this.ordsIter = ordsIter; + this.ord = ordsIter.next(); + return this; + } + + @Override + public boolean hasNext() { + return ord != 0; + } + + @Override + public short next() { + short value = values[ord]; + ord = ordsIter.next(); + return value; + } + } + } + + static class LongValues implements org.elasticsearch.index.fielddata.LongValues { + + private final short[] values; + private final Ordinals.Docs ordinals; + + private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1); + private final ValuesIter iter; + + LongValues(short[] values, Ordinals.Docs ordinals) { + this.values = values; + this.ordinals = ordinals; + this.iter = new ValuesIter(values); + } + + @Override + public boolean isMultiValued() { + return ordinals.isMultiValued(); + } + + @Override + public boolean hasValue(int docId) { + return ordinals.getOrd(docId) != 0; + } + + @Override + public long getValue(int docId) { + return (long) values[ordinals.getOrd(docId)]; + } + + @Override + public long getValueMissing(int docId, long missingValue) { + int ord = ordinals.getOrd(docId); + if (ord == 0) { + return missingValue; + } else { + return (long) values[ord]; + } + } + + @Override + public LongArrayRef getValues(int docId) { + IntArrayRef ords = ordinals.getOrds(docId); + int size = ords.size(); + if (size == 0) return LongArrayRef.EMPTY; + + arrayScratch.reset(size); + for (int i = ords.start; i < ords.end; i++) { + arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]]; + } + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset(ordinals.getIter(docId)); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + Ordinals.Docs.Iter iter = ordinals.getIter(docId); + int ord = iter.next(); + if (ord == 0) { + proc.onMissing(docId); + return; + } + do { + proc.onValue(docId, (long) values[ord]); + } while ((ord = iter.next()) != 0); + } + + static class ValuesIter implements Iter { + + private final short[] values; + private Ordinals.Docs.Iter ordsIter; + private int ord; + + ValuesIter(short[] values) { + this.values = values; + } + + public ValuesIter reset(Ordinals.Docs.Iter ordsIter) { + this.ordsIter = ordsIter; + this.ord = ordsIter.next(); + return this; + } + + @Override + public boolean hasNext() { + return ord != 0; + } + + @Override + public long next() { + short value = values[ord]; + ord = ordsIter.next(); + return (long) value; + } + } + } + + static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues { + + private final short[] values; + private final Ordinals.Docs ordinals; + + private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1); + private final ValuesIter iter; + + DoubleValues(short[] values, Ordinals.Docs ordinals) { + this.values = values; + this.ordinals = ordinals; + this.iter = new ValuesIter(values); + } + + @Override + public boolean isMultiValued() { + return ordinals.isMultiValued(); + } + + @Override + public boolean hasValue(int docId) { + return ordinals.getOrd(docId) != 0; + } + + @Override + public double getValue(int docId) { + return (double) values[ordinals.getOrd(docId)]; + } + + @Override + public double getValueMissing(int docId, double missingValue) { + int ord = ordinals.getOrd(docId); + if (ord == 0) { + return missingValue; + } else { + return (double) values[ord]; + } + } + + @Override + public DoubleArrayRef getValues(int docId) { + IntArrayRef ords = ordinals.getOrds(docId); + int size = ords.size(); + if (size == 0) return DoubleArrayRef.EMPTY; + + arrayScratch.reset(size); + for (int i = ords.start; i < ords.end; i++) { + arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]]; + } + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset(ordinals.getIter(docId)); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + Ordinals.Docs.Iter iter = ordinals.getIter(docId); + int ord = iter.next(); + if (ord == 0) { + proc.onMissing(docId); + return; + } + do { + proc.onValue(docId, (double) values[ord]); + } while ((ord = iter.next()) != 0); + } + + static class ValuesIter implements Iter { + + private final short[] values; + private Ordinals.Docs.Iter ordsIter; + private int ord; + + ValuesIter(short[] values) { + this.values = values; + } + + public ValuesIter reset(Ordinals.Docs.Iter ordsIter) { + this.ordsIter = ordsIter; + this.ord = ordsIter.next(); + return this; + } + + @Override + public boolean hasNext() { + return ord != 0; + } + + @Override + public double next() { + short value = values[ord]; + ord = ordsIter.next(); + return (double) value; + } + } + } + } + + /** + * A single valued case, where not all values are "set", so we have a FixedBitSet that + * indicates which values have an actual value. + */ + public static class SingleFixedSet extends ShortArrayAtomicFieldData { + + private final FixedBitSet set; + + public SingleFixedSet(short[] values, int numDocs, FixedBitSet set) { + super(values, numDocs); + this.set = set; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean isValuesOrdered() { + return false; + } + + @Override + public long getMemorySizeInBytes() { + if (size == -1) { + size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT) + (set.getBits().length * RamUsage.NUM_BYTES_LONG); + } + return size; + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.NumericDouble(getDoubleValues()); + } + + @Override + public BytesValues getBytesValues() { + return new BytesValues.StringBased(getStringValues()); + } + + @Override + public HashedBytesValues getHashedBytesValues() { + return new HashedBytesValues.StringBased(getStringValues()); + } + + @Override + public StringValues getStringValues() { + return new StringValues.ShortBased(getShortValues()); + } + + @Override + public ByteValues getByteValues() { + return new ByteValues.LongBased(getLongValues()); + } + + @Override + public ShortValues getShortValues() { + return new ShortValues(values, set); + } + + @Override + public IntValues getIntValues() { + return new IntValues.LongBased(getLongValues()); + } + + @Override + public LongValues getLongValues() { + return new LongValues(values, set); + } + + @Override + public FloatValues getFloatValues() { + return new FloatValues.DoubleBased(getDoubleValues()); + } + + @Override + public DoubleValues getDoubleValues() { + return new DoubleValues(values, set); + } + + static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues { + + private final short[] values; + private final FixedBitSet set; + + private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1); + private final Iter.Single iter = new Iter.Single(); + + ShortValues(short[] values, FixedBitSet set) { + this.values = values; + this.set = set; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return set.get(docId); + } + + @Override + public short getValue(int docId) { + return values[docId]; + } + + @Override + public short getValueMissing(int docId, short missingValue) { + if (set.get(docId)) { + return values[docId]; + } else { + return missingValue; + } + } + + @Override + public ShortArrayRef getValues(int docId) { + if (set.get(docId)) { + arrayScratch.values[0] = values[docId]; + return arrayScratch; + } else { + return ShortArrayRef.EMPTY; + } + } + + @Override + public Iter getIter(int docId) { + if (set.get(docId)) { + return iter.reset(values[docId]); + } else { + return Iter.Empty.INSTANCE; + } + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + if (set.get(docId)) { + proc.onValue(docId, values[docId]); + } + } + } + + static class LongValues implements org.elasticsearch.index.fielddata.LongValues { + + private final short[] values; + private final FixedBitSet set; + + private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1); + private final Iter.Single iter = new Iter.Single(); + + LongValues(short[] values, FixedBitSet set) { + this.values = values; + this.set = set; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return set.get(docId); + } + + @Override + public long getValue(int docId) { + return (long) values[docId]; + } + + @Override + public long getValueMissing(int docId, long missingValue) { + if (set.get(docId)) { + return (long) values[docId]; + } else { + return missingValue; + } + } + + @Override + public LongArrayRef getValues(int docId) { + if (set.get(docId)) { + arrayScratch.values[0] = (long) values[docId]; + return arrayScratch; + } else { + return LongArrayRef.EMPTY; + } + } + + @Override + public Iter getIter(int docId) { + if (set.get(docId)) { + return iter.reset((long) values[docId]); + } else { + return Iter.Empty.INSTANCE; + } + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + if (set.get(docId)) { + proc.onValue(docId, (long) values[docId]); + } + } + } + + static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues { + + private final short[] values; + private final FixedBitSet set; + + private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1); + private final Iter.Single iter = new Iter.Single(); + + DoubleValues(short[] values, FixedBitSet set) { + this.values = values; + this.set = set; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return set.get(docId); + } + + @Override + public double getValue(int docId) { + return (double) values[docId]; + } + + @Override + public double getValueMissing(int docId, double missingValue) { + if (set.get(docId)) { + return (double) values[docId]; + } else { + return missingValue; + } + } + + @Override + public DoubleArrayRef getValues(int docId) { + if (set.get(docId)) { + arrayScratch.values[0] = (double) values[docId]; + return arrayScratch; + } else { + return DoubleArrayRef.EMPTY; + } + } + + @Override + public Iter getIter(int docId) { + if (set.get(docId)) { + return iter.reset((double) values[docId]); + } else { + return Iter.Empty.INSTANCE; + } + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + if (set.get(docId)) { + proc.onValue(docId, (double) values[docId]); + } + } + } + + } + + /** + * Assumes all the values are "set", and docId is used as the index to the value array. + */ + public static class Single extends ShortArrayAtomicFieldData { + + /** + * Note, here, we assume that there is no offset by 1 from docId, so position 0 + * is the value for docId 0. + */ + public Single(short[] values, int numDocs) { + super(values, numDocs); + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean isValuesOrdered() { + return false; + } + + @Override + public long getMemorySizeInBytes() { + if (size == -1) { + size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT); + } + return size; + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.NumericDouble(getDoubleValues()); + } + + @Override + public BytesValues getBytesValues() { + return new BytesValues.StringBased(getStringValues()); + } + + @Override + public HashedBytesValues getHashedBytesValues() { + return new HashedBytesValues.StringBased(getStringValues()); + } + + @Override + public StringValues getStringValues() { + return new StringValues.ShortBased(getShortValues()); + } + + @Override + public ByteValues getByteValues() { + return new ByteValues.LongBased(getLongValues()); + } + + @Override + public ShortValues getShortValues() { + return new ShortValues(values); + } + + @Override + public IntValues getIntValues() { + return new IntValues.LongBased(getLongValues()); + } + + @Override + public LongValues getLongValues() { + return new LongValues(values); + } + + @Override + public FloatValues getFloatValues() { + return new FloatValues.DoubleBased(getDoubleValues()); + } + + @Override + public DoubleValues getDoubleValues() { + return new DoubleValues(values); + } + + static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues { + + private final short[] values; + + private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1); + private final Iter.Single iter = new Iter.Single(); + + ShortValues(short[] values) { + this.values = values; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return true; + } + + @Override + public short getValue(int docId) { + return values[docId]; + } + + @Override + public short getValueMissing(int docId, short missingValue) { + return values[docId]; + } + + @Override + public ShortArrayRef getValues(int docId) { + arrayScratch.values[0] = values[docId]; + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset(values[docId]); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + proc.onValue(docId, values[docId]); + } + } + + static class LongValues implements org.elasticsearch.index.fielddata.LongValues { + + private final short[] values; + + private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1); + private final Iter.Single iter = new Iter.Single(); + + LongValues(short[] values) { + this.values = values; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return true; + } + + @Override + public long getValue(int docId) { + return (long) values[docId]; + } + + @Override + public long getValueMissing(int docId, long missingValue) { + return (long) values[docId]; + } + + @Override + public LongArrayRef getValues(int docId) { + arrayScratch.values[0] = (long) values[docId]; + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset((long) values[docId]); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + proc.onValue(docId, (long) values[docId]); + } + } + + static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues { + + private final short[] values; + + private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1); + private final Iter.Single iter = new Iter.Single(); + + DoubleValues(short[] values) { + this.values = values; + } + + @Override + public boolean isMultiValued() { + return false; + } + + @Override + public boolean hasValue(int docId) { + return true; + } + + @Override + public double getValue(int docId) { + return (double) values[docId]; + } + + @Override + public double getValueMissing(int docId, double missingValue) { + return (double) values[docId]; + } + + @Override + public DoubleArrayRef getValues(int docId) { + arrayScratch.values[0] = (double) values[docId]; + return arrayScratch; + } + + @Override + public Iter getIter(int docId) { + return iter.reset((double) values[docId]); + } + + @Override + public void forEachValueInDoc(int docId, ValueInDocProc proc) { + proc.onValue(docId, (double) values[docId]); + } + } + } +} diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java new file mode 100644 index 00000000000..d1f7c107ac4 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java @@ -0,0 +1,157 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.plain; + +import gnu.trove.list.array.TShortArrayList; +import org.apache.lucene.index.*; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; +import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.settings.IndexSettings; + +import java.util.ArrayList; + +/** + */ +public class ShortArrayIndexFieldData extends AbstractIndexFieldData implements IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + + @Override + public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) { + return new ShortArrayIndexFieldData(index, indexSettings, fieldNames, type, cache); + } + } + + public ShortArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) { + super(index, indexSettings, fieldNames, fieldDataType, cache); + } + + @Override + public NumericType getNumericType() { + return NumericType.SHORT; + } + + @Override + public boolean valuesOrdered() { + // because we might have single values? we can dynamically update a flag to reflect that + // based on the atomic field data loaded + return false; + } + + @Override + public ShortArrayAtomicFieldData load(AtomicReaderContext context) { + try { + return cache.load(context, this); + } catch (Throwable e) { + if (e instanceof ElasticSearchException) { + throw (ElasticSearchException) e; + } else { + throw new ElasticSearchException(e.getMessage(), e); + } + } + } + + @Override + public ShortArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { + AtomicReader reader = context.reader(); + + Terms terms = reader.terms(getFieldNames().indexName()); + if (terms == null) { + return new ShortArrayAtomicFieldData.Single(new short[0], 0); + } + + // TODO: how can we guess the number of terms? numerics end up creating more terms per value... + final TShortArrayList values = new TShortArrayList(); + ArrayList ordinals = new ArrayList(); + int[] idx = new int[reader.maxDoc()]; + ordinals.add(new int[reader.maxDoc()]); + + values.add((short) 0); // first "t" indicates null value + int termOrd = 1; // current term number + + TermsEnum termsEnum = terms.iterator(null); + try { + DocsEnum docsEnum = null; + for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { + values.add((short) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term)); + docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0); + for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) { + int[] ordinal; + if (idx[docId] >= ordinals.size()) { + ordinal = new int[reader.maxDoc()]; + ordinals.add(ordinal); + } else { + ordinal = ordinals.get(idx[docId]); + } + ordinal[docId] = termOrd; + idx[docId]++; + } + termOrd++; + } + } catch (RuntimeException e) { + if (e.getClass().getName().endsWith("StopFillCacheException")) { + // all is well, in case numeric parsers are used. + } else { + throw e; + } + } + + if (ordinals.size() == 1) { + int[] nativeOrdinals = ordinals.get(0); + FixedBitSet set = new FixedBitSet(reader.maxDoc()); + short[] sValues = new short[reader.maxDoc()]; + boolean allHaveValue = true; + for (int i = 0; i < nativeOrdinals.length; i++) { + int nativeOrdinal = nativeOrdinals[i]; + if (nativeOrdinal == 0) { + allHaveValue = false; + } else { + set.set(i); + sValues[i] = values.get(nativeOrdinal); + } + } + if (allHaveValue) { + return new ShortArrayAtomicFieldData.Single(sValues, reader.maxDoc()); + } else { + return new ShortArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set); + } + } else { + int[][] nativeOrdinals = new int[ordinals.size()][]; + for (int i = 0; i < nativeOrdinals.length; i++) { + nativeOrdinals[i] = ordinals.get(i); + } + return new ShortArrayAtomicFieldData.WithOrdinals(values.toArray(new short[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd)); + } + } + + @Override + public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) { + return new DoubleValuesComparatorSource(this, missingValue); + } +} diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 729aafc6894..7c410ad093c 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -133,7 +133,7 @@ public class ShortFieldMapper extends NumberFieldMapper { @Override public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() { - throw new ElasticSearchIllegalArgumentException("not implemented"); + return new org.elasticsearch.index.fielddata.FieldDataType("short"); } @Override diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java new file mode 100644 index 00000000000..6ca9a5a9eca --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java @@ -0,0 +1,37 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.index.fielddata; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.StringField; +import org.elasticsearch.index.fielddata.FieldDataType; + +/** + */ +public class ShortFieldDataTests extends IntFieldDataTests { + + @Override + protected FieldDataType getFieldDataType() { + return new FieldDataType("short"); + } + +}