honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fielddata: Switch to Lucene DV APIs. 

This commits removes BytesValues/LongValues/DoubleValues/... and tries to use
Lucene's APIs such as NumericDocValues or RandomAccessOrds instead whenever
possible.

The next step would be to take advantage of the fact that APIs are the same in
Lucene and Elasticsearch in order to remove our custom comparators and use
Lucene's.

There are a few side-effects to this change:
 - GeoDistanceComparator has been removed, DoubleValuesComparator is used instead
   on top of dynamically computed values (was easier than migrating
   GeoDistanceComparator).
 - SortedNumericDocValues doesn't guarantee uniqueness so long/double terms
   aggregators have been updated to make sure a document cannot fall twice in
   the same bucket.
 - Sorting by maximum value of a field or running a `max` aggregation is
   potentially significantly faster thanks to the random-access API.

Our aggs and p/c aggregations benchmarks don't report differences with this
change on uninverted field data. However the fact that doc values don't need
to be wrapped anymore seems to help a lot. For example
TermsAggregationSearchBenchmark reports ~30% faster terms aggregations on doc
values on string fields with this change, which are now only ~18% slower than
uninverted field data although stored on disk.

Close #6908
This commit is contained in:
Adrien Grand 2014-07-03 01:07:47 +02:00
parent 0de30e1798
commit 3c142e550d
202 changed files with 4969 additions and 6299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/java/org/elasticsearch/common/geo/GeoDistance.java
View File

@ -19,9 +19,11 @@
package org.elasticsearch.common.geo;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.SloppyMath;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.fielddata.*;
import java.util.Locale;
@ -368,4 +370,42 @@ public enum GeoDistance {
return SLOPPY_ARC.calculate(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude, unit);
}
}
/**
* Return a {@link SortedNumericDoubleValues} instance that returns the distance to a given geo-point for each document.
*/
public static SortedNumericDoubleValues distanceValues(final FixedSourceDistance distance, final MultiGeoPointValues geoPointValues) {
final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues);
if (singleValues != null) {
final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues);
return FieldData.singleton(new NumericDoubleValues() {
@Override
public double get(int docID) {
if (docsWithField != null && !docsWithField.get(docID)) {
return 0d;
}
final GeoPoint point = singleValues.get(docID);
return distance.calculate(point.lat(), point.lon());
}
}, docsWithField);
} else {
return new SortingNumericDoubleValues() {
@Override
public void setDocument(int doc) {
geoPointValues.setDocument(doc);
count = geoPointValues.count();
grow();
for (int i = 0; i < count; ++i) {
final GeoPoint point = geoPointValues.valueAt(i);
values[i] = distance.calculate(point.lat(), point.lon());
}
sort();
}
};
}
}
}

9
src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java
View File

@ -22,8 +22,8 @@ package org.elasticsearch.common.lucene.search.function;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Explanation;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import java.util.Locale;
@ -37,7 +37,7 @@ public class FieldValueFactorFunction extends ScoreFunction {
private final float boostFactor;
private final Modifier modifier;
private final IndexNumericFieldData indexFieldData;
private DoubleValues values;
private SortedNumericDoubleValues values;
public FieldValueFactorFunction(String field, float boostFactor, Modifier modifierType, IndexNumericFieldData indexFieldData) {
super(CombineFunction.MULT);
@ -54,9 +54,10 @@ public class FieldValueFactorFunction extends ScoreFunction {
@Override
public double score(int docId, float subQueryScore) {
final int numValues = this.values.setDocument(docId);
this.values.setDocument(docId);
final int numValues = this.values.count();
if (numValues > 0) {
double val = this.values.nextValue() * boostFactor;
double val = this.values.valueAt(0) * boostFactor;
double result = modifier.apply(val);
if (Double.isNaN(result) || Double.isInfinite(result)) {
throw new ElasticsearchException("Result of field modification [" + modifier.toString() +

90
src/main/java/org/elasticsearch/index/fielddata/AbstractAtomicNumericFieldData.java
View File

@ -1,90 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
/**
*/
public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFieldData {
private boolean isFloat;
public AbstractAtomicNumericFieldData(boolean isFloat) {
this.isFloat = isFloat;
}
@Override
public ScriptDocValues getScriptValues() {
if (isFloat) {
return new ScriptDocValues.Doubles(getDoubleValues());
} else {
return new ScriptDocValues.Longs(getLongValues());
}
}
@Override
public BytesValues getBytesValues() {
if (isFloat) {
final DoubleValues values = getDoubleValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public BytesRef nextValue() {
scratch.copyChars(Double.toString(values.nextValue()));
return scratch;
}
@Override
public Order getOrder() {
return values.getOrder();
}
};
} else {
final LongValues values = getLongValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public BytesRef nextValue() {
scratch.copyChars(Long.toString(values.nextValue()));
return scratch;
}
@Override
public Order getOrder() {
return values.getOrder();
}
};
}
}
}

27
src/main/java/org/elasticsearch/index/fielddata/plain/DenseDoubleValues.java → src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java
View File

@ -16,22 +16,29 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.elasticsearch.index.fielddata.DoubleValues;
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.RandomAccessOrds;
/**
* Package private base class for dense double values.
* Base implementation of a {@link RandomAccessOrds} instance.
*/
abstract class DenseDoubleValues extends DoubleValues {
public abstract class AbstractRandomAccessOrds extends RandomAccessOrds {
protected DenseDoubleValues(boolean multiValued) {
super(multiValued);
int i = 0;
protected abstract void doSetDocument(int docID);
@Override
public final void setDocument(int docID) {
doSetDocument(docID);
i = 0;
}
@Override
public final int setDocument(int docId) {
this.docId = docId;
return 1;
public long nextOrd() {
return ordAt(i++);
}
}
}

106
src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
View File

@ -20,117 +20,21 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
import org.elasticsearch.common.lease.Releasable;
/**
* The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
*/
public interface AtomicFieldData<Script extends ScriptDocValues> extends Accountable {
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
BytesValues getBytesValues();
public interface AtomicFieldData extends Accountable, Releasable {
/**
* Returns a "scripting" based values.
*/
Script getScriptValues();
ScriptDocValues getScriptValues();
/**
* Close the field data.
* Return a String representation of the values.
*/
void close();
SortedBinaryDocValues getBytesValues();
interface WithOrdinals<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
public static final WithOrdinals<ScriptDocValues.Strings> EMPTY = new WithOrdinals<ScriptDocValues.Strings>() {
@Override
public Strings getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public void close() {
}
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new BytesValues.WithOrdinals(false) {
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public long nextOrd() {
return MISSING_ORDINAL;
}
@Override
public BytesRef getValueByOrd(long ord) {
throw new UnsupportedOperationException();
}
@Override
public long getOrd(int docId) {
return MISSING_ORDINAL;
}
@Override
public long getMaxOrd() {
return 0;
}
};
}
};
/**
* Use a non thread safe (lightweight) view of the values as bytes.
* @param needsHashes
*/
BytesValues.WithOrdinals getBytesValues();
}
/**
* This enum provides information about the order of the values for
* a given document. For instance {@link BytesValues} by default
* return values in {@link #BYTES} order but if the interface
* wraps a numeric variant the sort order might change to {@link #NUMERIC}.
* In that case the values might not be returned in byte sort order but in numeric
* order instead while maintaining the property of <tt>N < N+1</tt> during the
* value iterations.
*
* @see org.elasticsearch.index.fielddata.BytesValues#getOrder()
* @see org.elasticsearch.index.fielddata.DoubleValues#getOrder()
* @see org.elasticsearch.index.fielddata.LongValues#getOrder()
*/
public enum Order {
/**
* Donates Byte sort order
*/
BYTES,
/**
* Donates Numeric sort order
*/
NUMERIC,
/**
* Donates custom sort order
*/
CUSTOM,
/**
* Donates no sort order
*/
NONE
}
}

32
src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
View File

@ -18,35 +18,15 @@
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.geo.GeoPoint;
/**
* {@link AtomicFieldData} specialization for geo points.
*/
public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> implements AtomicFieldData<Script> {
public interface AtomicGeoPointFieldData extends AtomicFieldData {
public abstract GeoPointValues getGeoPointValues();
@Override
public BytesValues getBytesValues() {
final GeoPointValues values = getGeoPointValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public BytesRef nextValue() {
GeoPoint value = values.nextValue();
scratch.copyChars(GeoHashUtils.encode(value.lat(), value.lon()));
return scratch;
}
};
}
/**
* Return geo-point values.
*/
MultiGeoPointValues getGeoPointValues();
}

19
src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
View File

@ -19,12 +19,25 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedNumericDocValues;
/**
* Specialization of {@link AtomicFieldData} for numeric data.
*/
public interface AtomicNumericFieldData extends AtomicFieldData<ScriptDocValues> {
public interface AtomicNumericFieldData extends AtomicFieldData {
LongValues getLongValues();
/**
* Get an integer view of the values of this segment. If the implementation
* stores floating-point numbers then these values will return the same
* values but casted to longs.
*/
SortedNumericDocValues getLongValues();
DoubleValues getDoubleValues();
/**
* Return a floating-point view of the values in this segment. If the
* implementation stored integers then the returned doubles would be the
* same ones as you would get from casting to a double.
*/
SortedNumericDoubleValues getDoubleValues();
}

24
src/main/java/org/elasticsearch/index/fielddata/plain/DenseLongValues.java → src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java
View File

@ -16,22 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.elasticsearch.index.fielddata.LongValues;
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.RandomAccessOrds;
/**
* Package private base class for dense long values.
* Specialization of {@link AtomicFieldData} for data that is indexed with
* ordinals.
*/
abstract class DenseLongValues extends LongValues {
public interface AtomicOrdinalsFieldData extends AtomicFieldData {
protected DenseLongValues(boolean multiValued) {
super(multiValued);
}
/**
* Return the ordinals values for the current atomic reader.
*/
RandomAccessOrds getOrdinalsValues();
@Override
public final int setDocument(int docId) {
this.docId = docId;
return 1;
}
}
}

41
src/main/java/org/elasticsearch/index/fielddata/FilterLongValues.java → src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java
View File

@ -16,35 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedDocValues;
import java.util.Set;
/**
* <code>FilterLongValues</code> contains another {@link LongValues}, which it
* uses as its basic source of data, possibly transforming the data along the
* way or providing additional functionality.
* Specialization of {@link AtomicFieldData} for parent/child mappings.
*/
public class FilterLongValues extends LongValues {
public interface AtomicParentChildFieldData extends AtomicFieldData {
protected final LongValues delegate;
/**
* Return the set of types there is a mapping for.
*/
Set<String> types();
protected FilterLongValues(LongValues delegate) {
super(delegate.isMultiValued());
this.delegate = delegate;
}
@Override
public int setDocument(int docId) {
return delegate.setDocument(docId);
}
@Override
public long nextValue() {
return delegate.nextValue();
}
@Override
public AtomicFieldData.Order getOrder() {
return delegate.getOrder();
}
/**
* Return the mapping for the given type. The returned
* {@link SortedDocValues} will map doc IDs to the identifier of their
* parent.
*/
SortedDocValues getOrdinalsValues(String type);
}

176
src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
View File

@ -1,176 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.plain.BytesValuesWithOrdinalsTermsEnum;
/**
* A state-full lightweight per document set of <code>byte[]</code> values.
*
* To iterate over values in a document use the following pattern:
* <pre>
* BytesValues values = ..;
* final int numValues = values.setDocId(docId);
* for (int i = 0; i < numValues; i++) {
* BytesRef value = values.nextValue();
* // process value
* }
* </pre>
*/
public abstract class BytesValues {
/**
* An empty {@link BytesValues instance}
*/
public static final BytesValues EMPTY = new Empty();
private final boolean multiValued;
/**
* Creates a new {@link BytesValues} instance
* @param multiValued <code>true</code> iff this instance is multivalued. Otherwise <code>false</code>.
*/
protected BytesValues(boolean multiValued) {
this.multiValued = multiValued;
}
/**
* Is one of the documents in this field data values is multi valued?
*/
public final boolean isMultiValued() {
return multiValued;
}
/**
* Sets iteration to the specified docID and returns the number of
* values for this document ID,
* @param docId document ID
*
* @see #nextValue()
*/
public abstract int setDocument(int docId);
/**
* Returns the next value for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined. This interface guarantees that the values are returned in order.
* <p>
* If this instance returns ordered values the <tt>Nth</tt> value is strictly less than the <tt>N+1</tt> value with
* respect to the {@link AtomicFieldData.Order} returned from {@link #getOrder()}. If this instance returns
* <i>unordered</i> values {@link #getOrder()} must return {@link AtomicFieldData.Order#NONE}
* Note: the values returned are de-duplicated, only unique values are returned.
* </p>
*
* Note: the returned {@link BytesRef} might be shared across invocations.
*
* @return the next value for the current docID set to {@link #setDocument(int)}.
*/
public abstract BytesRef nextValue();
/**
* Returns the order the values are returned from {@link #nextValue()}.
* <p> Note: {@link BytesValues} have {@link AtomicFieldData.Order#BYTES} by default.</p>
*/
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.BYTES;
}
/**
* Ordinal based {@link BytesValues}.
*/
public static abstract class WithOrdinals extends BytesValues {
public static final long MIN_ORDINAL = 0;
public static final long MISSING_ORDINAL = SortedSetDocValues.NO_MORE_ORDS;
protected WithOrdinals(boolean multiValued) {
super(multiValued);
}
/**
* Returns total unique ord count;
*/
public abstract long getMaxOrd();
/**
* The ordinal that maps to the relevant docId. If it has no value, returns
* <tt>0</tt>.
*/
public abstract long getOrd(int docId);
/**
* Returns the next ordinal for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
*
* Note: This method will never return <tt>0</tt>.
*
* @return the next ordinal for the current docID set to {@link #setDocument(int)}.
*/
public abstract long nextOrd();
/**
* Returns the value for the given ordinal.
* @param ord the ordinal to lookup.
* @return a shared {@link BytesRef} instance holding the value associated
* with the given ordinal or <code>null</code> if ordinal is <tt>0</tt>
*/
public abstract BytesRef getValueByOrd(long ord);
@Override
public BytesRef nextValue() {
return getValueByOrd(nextOrd());
}
/**
* Returns a terms enum to iterate over all the underlying values.
*/
public TermsEnum getTermsEnum() {
return new BytesValuesWithOrdinalsTermsEnum(this);
}
}
/**
* An empty {@link BytesValues} implementation
*/
private final static class Empty extends BytesValues {
Empty() {
super(false);
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public BytesRef nextValue() {
throw new ElasticsearchIllegalStateException("Empty BytesValues has no next value");
}
}
}

164
src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
View File

@ -1,164 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticsearchIllegalStateException;
/**
* A state-full lightweight per document set of <code>double</code> values.
*
* To iterate over values in a document use the following pattern:
* <pre>
* DoubleValues values = ..;
* final int numValues = values.setDocId(docId);
* for (int i = 0; i < numValues; i++) {
* double value = values.nextValue();
* // process value
* }
* </pre>
*/
public abstract class DoubleValues {
/**
* An empty {@link DoubleValues instance}
*/
public static final DoubleValues EMPTY = new Empty();
private final boolean multiValued;
protected int docId;
/**
* Creates a new {@link DoubleValues} instance
* @param multiValued <code>true</code> iff this instance is multivalued. Otherwise <code>false</code>.
*/
protected DoubleValues(boolean multiValued) {
this.multiValued = multiValued;
}
/**
* Is one of the documents in this field data values is multi valued?
*/
public final boolean isMultiValued() {
return multiValued;
}
/**
* Sets iteration to the specified docID and returns the number of
* values for this document ID,
* @param docId document ID
*
* @see #nextValue()
*/
public abstract int setDocument(int docId);
/**
* Returns the next value for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
* <p>
* If this instance returns ordered values the <tt>Nth</tt> value is strictly less than the <tt>N+1</tt> value with
* respect to the {@link AtomicFieldData.Order} returned from {@link #getOrder()}. If this instance returns
* <i>unordered</i> values {@link #getOrder()} must return {@link AtomicFieldData.Order#NONE}
* Note: the values returned are de-duplicated, only unique values are returned.
* </p>
*
* @return the next value for the current docID set to {@link #setDocument(int)}.
*/
public abstract double nextValue();
/**
* Returns the order the values are returned from {@link #nextValue()}.
* <p> Note: {@link DoubleValues} have {@link AtomicFieldData.Order#NUMERIC} by default.</p>
*/
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NUMERIC;
}
/**
* Ordinal based {@link DoubleValues}.
*/
public static abstract class WithOrdinals extends DoubleValues {
protected final BytesValues.WithOrdinals ordinals;
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.ordinals = ordinals;
}
/**
* Returns the value for the given ordinal.
* @param ord the ordinal to lookup.
* @return a double value associated with the given ordinal.
*/
public abstract double getValueByOrd(long ord);
@Override
public int setDocument(int docId) {
this.docId = docId;
return ordinals.setDocument(docId);
}
@Override
public double nextValue() {
return getValueByOrd(ordinals.nextOrd());
}
}
/**
* An empty {@link DoubleValues} implementation
*/
private static class Empty extends DoubleValues {
Empty() {
super(false);
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public double nextValue() {
throw new ElasticsearchIllegalStateException("Empty DoubleValues has no next value");
}
}
/** Wrap a {@link LongValues} instance. */
public static DoubleValues asDoubleValues(final LongValues values) {
return new DoubleValues(values.isMultiValued()) {
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public double nextValue() {
return (double) values.nextValue();
}
};
}
}

533
src/main/java/org/elasticsearch/index/fielddata/FieldData.java Normal file
View File

@ -0,0 +1,533 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.lucene.Lucene;
import java.util.ArrayList;
import java.util.List;
/**
* Utility methods, similar to Lucene's {@link DocValues}.
*/
public enum FieldData {
;
static {
assert Lucene.VERSION == Version.LUCENE_4_9 : "Remove emptySortedNumeric in 4.10 and use the method with the same name from Lucene's DocValues class. See LUCENE-5834.";
}
/**
* Return a {@link SortedNumericDocValues} that doesn't contain any value.
*/
public static SortedNumericDocValues emptySortedNumeric(int maxDoc) {
return DocValues.singleton(DocValues.emptyNumeric(), new Bits.MatchNoBits(maxDoc));
}
/**
* Return a {@link NumericDoubleValues} that doesn't contain any value.
*/
public static NumericDoubleValues emptyNumericDouble() {
return new NumericDoubleValues() {
@Override
public double get(int docID) {
return 0;
}
};
}
/**
* Return a {@link SortedNumericDoubleValues} that doesn't contain any value.
*/
public static SortedNumericDoubleValues emptySortedNumericDoubles(int maxDoc) {
return singleton(emptyNumericDouble(), new Bits.MatchNoBits(maxDoc));
}
public static GeoPointValues emptyGeoPoint() {
final GeoPoint point = new GeoPoint();
return new GeoPointValues() {
@Override
public GeoPoint get(int docID) {
return point;
}
};
}
/**
* Return a {@link SortedNumericDoubleValues} that doesn't contain any value.
*/
public static MultiGeoPointValues emptyMultiGeoPoints(int maxDoc) {
return singleton(emptyGeoPoint(), new Bits.MatchNoBits(maxDoc));
}
/**
* Returns a {@link Bits} representing all documents from <code>dv</code> that have a value.
*/
public static Bits docsWithValue(final SortedBinaryDocValues dv, final int maxDoc) {
return new Bits() {
@Override
public boolean get(int index) {
dv.setDocument(index);
return dv.count() != 0;
}
@Override
public int length() {
return maxDoc;
}
};
}
/**
* Returns a Bits representing all documents from <code>dv</code> that have a value.
*/
public static Bits docsWithValue(final MultiGeoPointValues dv, final int maxDoc) {
return new Bits() {
@Override
public boolean get(int index) {
dv.setDocument(index);
return dv.count() != 0;
}
@Override
public int length() {
return maxDoc;
}
};
}
/**
* Returns a Bits representing all documents from <code>dv</code> that have a value.
*/
public static Bits docsWithValue(final SortedNumericDoubleValues dv, final int maxDoc) {
return new Bits() {
@Override
public boolean get(int index) {
dv.setDocument(index);
return dv.count() != 0;
}
@Override
public int length() {
return maxDoc;
}
};
}
/**
* Wrap the provided {@link SortedNumericDocValues} instance to cast all values to doubles.
*/
public static SortedNumericDoubleValues castToDouble(final SortedNumericDocValues values) {
final NumericDocValues singleton = DocValues.unwrapSingleton(values);
if (singleton != null) {
final Bits docsWithField = DocValues.unwrapSingletonBits(values);
return singleton(new DoubleCastedValues(singleton), docsWithField);
} else {
return new SortedDoubleCastedValues(values);
}
}
/**
* Wrap the provided {@link SortedNumericDoubleValues} instance to cast all values to longs.
*/
public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues values) {
final NumericDoubleValues singleton = unwrapSingleton(values);
if (singleton != null) {
final Bits docsWithField = unwrapSingletonBits(values);
return DocValues.singleton(new LongCastedValues(singleton), docsWithField);
} else {
return new SortedLongCastedValues(values);
}
}
/**
* Returns a multi-valued view over the provided {@link NumericDoubleValues}.
*/
public static SortedNumericDoubleValues singleton(NumericDoubleValues values, Bits docsWithField) {
return new SingletonSortedNumericDoubleValues(values, docsWithField);
}
/**
* Returns a single-valued view of the {@link SortedNumericDoubleValues},
* if it was previously wrapped with {@link #singleton(NumericDocValues, Bits)},
* or null.
* @see #unwrapSingletonBits(SortedNumericDocValues)
*/
public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues values) {
if (values instanceof SingletonSortedNumericDoubleValues) {
return ((SingletonSortedNumericDoubleValues) values).getNumericDoubleValues();
}
return null;
}
/**
* Returns the documents with a value for the {@link SortedNumericDoubleValues},
* if it was previously wrapped with {@link #singleton(NumericDoubleValues, Bits)},
* or null.
*/
public static Bits unwrapSingletonBits(SortedNumericDoubleValues dv) {
if (dv instanceof SingletonSortedNumericDoubleValues) {
return ((SingletonSortedNumericDoubleValues)dv).getDocsWithField();
} else {
return null;
}
}
/**
* Returns a multi-valued view over the provided {@link GeoPointValues}.
*/
public static MultiGeoPointValues singleton(GeoPointValues values, Bits docsWithField) {
return new SingletonMultiGeoPointValues(values, docsWithField);
}
/**
* Returns a single-valued view of the {@link MultiGeoPointValues},
* if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)},
* or null.
* @see #unwrapSingletonBits(MultiGeoPointValues)
*/
public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) {
if (values instanceof SingletonMultiGeoPointValues) {
return ((SingletonMultiGeoPointValues) values).getGeoPointValues();
}
return null;
}
/**
* Returns the documents with a value for the {@link MultiGeoPointValues},
* if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)},
* or null.
*/
public static Bits unwrapSingletonBits(MultiGeoPointValues values) {
if (values instanceof SingletonMultiGeoPointValues) {
return ((SingletonMultiGeoPointValues) values).getDocsWithField();
}
return null;
}
/**
* Returns a multi-valued view over the provided {@link BinaryDocValues}.
*/
public static SortedBinaryDocValues singleton(BinaryDocValues values, Bits docsWithField) {
return new SingletonSortedBinaryDocValues(values, docsWithField);
}
/**
* Returns a single-valued view of the {@link SortedBinaryDocValues},
* if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)},
* or null.
* @see #unwrapSingletonBits(SortedBinaryDocValues)
*/
public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) {
if (values instanceof SingletonSortedBinaryDocValues) {
return ((SingletonSortedBinaryDocValues) values).getBinaryDocValues();
}
return null;
}
/**
* Returns the documents with a value for the {@link SortedBinaryDocValues},
* if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)},
* or null.
*/
public static Bits unwrapSingletonBits(SortedBinaryDocValues values) {
if (values instanceof SingletonSortedBinaryDocValues) {
return ((SingletonSortedBinaryDocValues) values).getDocsWithField();
}
return null;
}
/**
* Returns whether the provided values *might* be multi-valued. There is no
* guarantee that this method will return <tt>false</tt> in the single-valued case.
*/
public static boolean isMultiValued(SortedSetDocValues values) {
return DocValues.unwrapSingleton(values) == null;
}
/**
* Returns whether the provided values *might* be multi-valued. There is no
* guarantee that this method will return <tt>false</tt> in the single-valued case.
*/
public static boolean isMultiValued(SortedNumericDocValues values) {
return DocValues.unwrapSingleton(values) == null;
}
/**
* Returns whether the provided values *might* be multi-valued. There is no
* guarantee that this method will return <tt>false</tt> in the single-valued case.
*/
public static boolean isMultiValued(SortedNumericDoubleValues values) {
return unwrapSingleton(values) == null;
}
/**
* Returns whether the provided values *might* be multi-valued. There is no
* guarantee that this method will return <tt>false</tt> in the single-valued case.
*/
public static boolean isMultiValued(SortedBinaryDocValues values) {
return unwrapSingleton(values) != null;
}
/**
* Returns whether the provided values *might* be multi-valued. There is no
* guarantee that this method will return <tt>false</tt> in the single-valued case.
*/
public static boolean isMultiValued(MultiGeoPointValues values) {
return unwrapSingleton(values) == null;
}
/**
* Return a {@link String} representation of the provided values. That is
* typically used for scripts or for the `map` execution mode of terms aggs.
* NOTE: this is very slow!
*/
public static SortedBinaryDocValues toString(final SortedNumericDocValues values) {
return toString(new ToStringValues() {
@Override
public void get(int docID, List<CharSequence> list) {
values.setDocument(docID);
for (int i = 0, count = values.count(); i < count; ++i) {
list.add(Long.toString(values.valueAt(i)));
}
}
});
}
/**
* Return a {@link String} representation of the provided values. That is
* typically used for scripts or for the `map` execution mode of terms aggs.
* NOTE: this is very slow!
*/
public static SortedBinaryDocValues toString(final SortedNumericDoubleValues values) {
return toString(new ToStringValues() {
@Override
public void get(int docID, List<CharSequence> list) {
values.setDocument(docID);
for (int i = 0, count = values.count(); i < count; ++i) {
list.add(Double.toString(values.valueAt(i)));
}
}
});
}
/**
* Return a {@link String} representation of the provided values. That is
* typically used for scripts or for the `map` execution mode of terms aggs.
* NOTE: this is very slow!
*/
public static SortedBinaryDocValues toString(final RandomAccessOrds values) {
return toString(new ToStringValues() {
@Override
public void get(int docID, List<CharSequence> list) {
values.setDocument(docID);
for (int i = 0, count = values.cardinality(); i < count; ++i) {
final long ord = values.ordAt(i);
list.add(values.lookupOrd(ord).utf8ToString());
}
}
});
}
/**
* Return a {@link String} representation of the provided values. That is
* typically used for scripts or for the `map` execution mode of terms aggs.
* NOTE: this is very slow!
*/
public static SortedBinaryDocValues toString(final MultiGeoPointValues values) {
return toString(new ToStringValues() {
@Override
public void get(int docID, List<CharSequence> list) {
values.setDocument(docID);
for (int i = 0, count = values.count(); i < count; ++i) {
list.add(values.valueAt(i).toString());
}
}
});
}
/**
* If <code>dv</code> is an instance of {@link RandomAccessOrds}, then return
* it, otherwise wrap it into a slow wrapper that implements random access.
*/
public static RandomAccessOrds maybeSlowRandomAccessOrds(final SortedSetDocValues dv) {
if (dv instanceof RandomAccessOrds) {
return (RandomAccessOrds) dv;
} else {
assert DocValues.unwrapSingleton(dv) == null : "this method expect singleton to return random-access ords";
return new RandomAccessOrds() {
int cardinality;
long[] ords = new long[0];
int ord;
@Override
public void setDocument(int docID) {
cardinality = 0;
dv.setDocument(docID);
for (long ord = dv.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = dv.nextOrd()) {
ords = ArrayUtil.grow(ords, cardinality + 1);
ords[cardinality++] = ord;
}
ord = 0;
}
@Override
public long nextOrd() {
return ords[ord++];
}
@Override
public BytesRef lookupOrd(long ord) {
return dv.lookupOrd(ord);
}
@Override
public long getValueCount() {
return dv.getValueCount();
}
@Override
public long ordAt(int index) {
return ords[index];
}
@Override
public int cardinality() {
return cardinality;
}
};
}
}
private static SortedBinaryDocValues toString(final ToStringValues toStringValues) {
return new SortingBinaryDocValues() {
final List<CharSequence> list = new ArrayList<>();
@Override
public void setDocument(int docID) {
list.clear();
toStringValues.get(docID, list);
count = list.size();
grow();
for (int i = 0; i < count; ++i) {
final CharSequence s = list.get(i);
values[i].copyChars(s);
}
sort();
}
};
}
private static interface ToStringValues {
void get(int docID, List<CharSequence> values);
}
private static class DoubleCastedValues extends NumericDoubleValues {
private final NumericDocValues values;
DoubleCastedValues(NumericDocValues values) {
this.values = values;
}
@Override
public double get(int docID) {
return values.get(docID);
}
}
private static class SortedDoubleCastedValues extends SortedNumericDoubleValues {
private final SortedNumericDocValues values;
SortedDoubleCastedValues(SortedNumericDocValues in) {
this.values = in;
}
@Override
public double valueAt(int index) {
return values.valueAt(index);
}
@Override
public void setDocument(int doc) {
values.setDocument(doc);
}
@Override
public int count() {
return values.count();
}
}
private static class LongCastedValues extends NumericDocValues {
private final NumericDoubleValues values;
LongCastedValues(NumericDoubleValues values) {
this.values = values;
}
@Override
public long get(int docID) {
return (long) values.get(docID);
}
}
private static class SortedLongCastedValues extends SortedNumericDocValues {
private final SortedNumericDoubleValues values;
SortedLongCastedValues(SortedNumericDoubleValues in) {
this.values = in;
}
@Override
public long valueAt(int index) {
return (long) values.valueAt(index);
}
@Override
public void setDocument(int doc) {
values.setDocument(doc);
}
@Override
public int count() {
return values.count();
}
}
}

91
src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
View File

@ -16,99 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.geo.GeoPoint;
/**
* A state-full lightweight per document set of {@link GeoPoint} values.
* To iterate over values in a document use the following pattern:
* <pre>
* GeoPointValues values = ..;
* final int numValues = values.setDocId(docId);
* for (int i = 0; i < numValues; i++) {
* GeoPoint value = values.nextValue();
* // process value
* }
* </pre>
* Per-document geo-point values.
*/
public abstract class GeoPointValues {
/**
* An empty {@link GeoPointValues instance}
* Get the {@link GeoPoint} associated with <code>docID</code>.
* The returned {@link GeoPoint} might be reused across calls.
* If the given <code>docID</code> does not have a value then the returned
* geo point mught have both latitude and longitude set to 0.
*/
public static final GeoPointValues EMPTY = new Empty();
private final boolean multiValued;
public abstract GeoPoint get(int docID);
protected int docId = -1;
/**
* Creates a new {@link GeoPointValues} instance
* @param multiValued <code>true</code> iff this instance is multivalued. Otherwise <code>false</code>.
*/
protected GeoPointValues(boolean multiValued) {
this.multiValued = multiValued;
}
/**
* Is one of the documents in this field data values is multi valued?
*/
public final boolean isMultiValued() {
return multiValued;
}
/**
* Sets iteration to the specified docID and returns the number of
* values for this document ID,
* @param docId document ID
*
* @see #nextValue()
*/
public abstract int setDocument(int docId);
/**
* Returns the next value for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
* <p>
* If this instance returns ordered values the <tt>Nth</tt> value is strictly less than the <tt>N+1</tt> value with
* respect to the {@link AtomicFieldData.Order} returned from {@link #getOrder()}. If this instance returns
* <i>unordered</i> values {@link #getOrder()} must return {@link AtomicFieldData.Order#NONE}
* Note: the values returned are de-duplicated, only unique values are returned.
* </p>
*
* Note: the returned {@link GeoPoint} might be shared across invocations.
*
* @return the next value for the current docID set to {@link #setDocument(int)}.
*/
public abstract GeoPoint nextValue();
/**
* Returns the order the values are returned from {@link #nextValue()}.
* <p> Note: {@link GeoPointValues} have {@link AtomicFieldData.Order#NONE} by default.</p>
*/
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NONE;
}
/**
* An empty {@link GeoPointValues} implementation
*/
private static final class Empty extends GeoPointValues {
protected Empty() {
super(false);
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public GeoPoint nextValue() {
throw new ElasticsearchIllegalStateException("Empty GeoPointValues has no next value");
}
}
}

30
src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
View File

@ -29,14 +29,15 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
/**
* Thread-safe utility class that allows to get per-segment values via the
* {@link #load(AtomicReaderContext)} method.
*/
public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
@ -79,11 +80,6 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
*/
FieldDataType getFieldDataType();
/**
* Are the values ordered? (in ascending manner).
*/
boolean valuesOrdered();
/**
* Loads the atomic field data for the reader, possibly cached.
*/
@ -195,25 +191,15 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
interface Builder {
IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder);
IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService);
}
public interface WithOrdinals<FD extends AtomicFieldData.WithOrdinals> extends IndexFieldData<FD> {
public static interface Global<FD extends AtomicFieldData> extends IndexFieldData<FD> {
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
IndexFieldData<FD> loadGlobal(IndexReader indexReader);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
WithOrdinals loadGlobal(IndexReader indexReader);
WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception;
IndexFieldData<FD> localGlobalDirect(IndexReader indexReader) throws Exception;
}

16
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
View File

@ -48,7 +48,7 @@ public interface IndexFieldDataCache {
<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;
<IFD extends IndexFieldData.WithOrdinals<?>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception;
<FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception;
/**
* Clears all the field data stored cached in on this index.
@ -78,7 +78,7 @@ public interface IndexFieldDataCache {
@Override
@SuppressWarnings("unchecked")
public <IFD extends IndexFieldData.WithOrdinals<?>> IFD load(IndexReader indexReader, IFD indexFieldData) throws Exception {
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(IndexReader indexReader, IFD indexFieldData) throws Exception {
return (IFD) indexFieldData.localGlobalDirect(indexReader);
}
@ -143,9 +143,9 @@ public interface IndexFieldDataCache {
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
final Key key = new Key(context.reader().getCoreCacheKey());
//noinspection unchecked
return (FD) cache.get(key, new Callable<AtomicFieldData>() {
Accountable fd = cache.get(key, new Callable<FD>() {
@Override
public AtomicFieldData call() throws Exception {
public FD call() throws Exception {
SegmentReaderUtils.registerCoreListener(context.reader(), FieldBased.this);
key.listeners.add(indicesFieldDataCacheListener);
@ -156,7 +156,7 @@ public interface IndexFieldDataCache {
key.listeners.add(shard.fieldData());
}
}
final AtomicFieldData fieldData = indexFieldData.loadDirect(context);
final FD fieldData = indexFieldData.loadDirect(context);
key.sizeInBytes = fieldData.ramBytesUsed();
for (Listener listener : key.listeners) {
try {
@ -169,12 +169,13 @@ public interface IndexFieldDataCache {
return fieldData;
}
});
return (FD) fd;
}
public <IFD extends IndexFieldData.WithOrdinals<?>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception {
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception {
final Key key = new Key(indexReader.getCoreCacheKey());
//noinspection unchecked
return (IFD) cache.get(key, new Callable<Accountable>() {
Accountable ifd = cache.get(key, new Callable<Accountable>() {
@Override
public GlobalOrdinalsIndexFieldData call() throws Exception {
indexReader.addReaderClosedListener(FieldBased.this);
@ -201,6 +202,7 @@ public interface IndexFieldDataCache {
return ifd;
}
});
return (IFD) ifd;
}
@Override

10
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View File

@ -32,8 +32,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.util.concurrent.KeyedLock;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.plain.*;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
@ -288,9 +286,8 @@ public class IndexFieldDataService extends AbstractIndexComponent {
fieldDataCaches.put(fieldNames.indexName(), cache);
}
GlobalOrdinalsBuilder globalOrdinalBuilder = new InternalGlobalOrdinalsBuilder(index(), indexSettings);
fieldData = builder.build(index, indexSettings, mapper, cache, circuitBreakerService, indexService.mapperService(), globalOrdinalBuilder);
loadedFieldData.put(key, fieldData);
fieldData = builder.build(index, indexSettings, mapper, cache, circuitBreakerService, indexService.mapperService());
loadedFieldData.put(fieldNames.indexName(), fieldData);
}
} finally {
fieldLoadingLock.release(key);
@ -330,9 +327,8 @@ public class IndexFieldDataService extends AbstractIndexComponent {
}
CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
GlobalOrdinalsBuilder globalOrdinalBuilder = new InternalGlobalOrdinalsBuilder(index(), indexSettings);
@SuppressWarnings("unchecked")
IFD ifd = (IFD) builder.build(index, indexSettings, mapper, new IndexFieldDataCache.None(), circuitBreakerService, indexService.mapperService(), globalOrdinalBuilder);
IFD ifd = (IFD) builder.build(index, indexSettings, mapper, new IndexFieldDataCache.None(), circuitBreakerService, indexService.mapperService());
return ifd;
}

14
src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
View File

@ -19,19 +19,9 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
/**
* Specialization of {@link IndexFieldData} for geo points.
*/
public interface IndexGeoPointFieldData<FD extends AtomicGeoPointFieldData> extends IndexFieldData<FD> {
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
public interface IndexGeoPointFieldData extends IndexFieldData<AtomicGeoPointFieldData> {
}

13
src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
@ -28,7 +27,7 @@ import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
/**
*/
public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extends IndexFieldData<FD> {
public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericFieldData> {
public static enum NumericType {
BYTE(8, false, SortField.Type.INT, Byte.MIN_VALUE, Byte.MAX_VALUE) {
@ -183,14 +182,4 @@ public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extend
}
NumericType getNumericType();
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
}

42
src/main/java/org/elasticsearch/index/fielddata/FilterDoubleValues.java → src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
View File

@ -16,35 +16,27 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.IndexReader;
/**
* <code>FilterDoubleValues</code> contains another {@link DoubleValues}, which it
* uses as its basic source of data, possibly transforming the data along the
* way or providing additional functionality.
* Specialization of {@link IndexFieldData} for data that is indexed with ordinals.
*/
public abstract class FilterDoubleValues extends DoubleValues {
public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdinalsFieldData> {
protected final DoubleValues delegate;
protected FilterDoubleValues(DoubleValues delegate) {
super(delegate.isMultiValued());
this.delegate = delegate;
}
@Override
public int setDocument(int docId) {
return delegate.setDocument(docId);
}
@Override
public double nextValue() {
return delegate.nextValue();
}
@Override
public AtomicFieldData.Order getOrder() {
return delegate.getOrder();
}
/**
* Load a global view of the ordinals for the given {@link IndexReader},
* potentially from a cache.
*/
IndexOrdinalsFieldData loadGlobal(IndexReader indexReader);
/**
* Load a global view of the ordinals for the given {@link IndexReader}.
*/
IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
}

43
src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.IndexReader;
/**
* Soecialization of {@link IndexFieldData} for parent/child mappings.
*/
public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicParentChildFieldData> {
/**
* Load a global view of the ordinals for the given {@link IndexReader},
* potentially from a cache.
*/
IndexParentChildFieldData loadGlobal(IndexReader indexReader);
/**
* Load a global view of the ordinals for the given {@link IndexReader}.
*/
IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
}

164
src/main/java/org/elasticsearch/index/fielddata/LongValues.java
View File

@ -1,164 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticsearchIllegalStateException;
/**
* A state-full lightweight per document set of <code>long</code> values.
*
* To iterate over values in a document use the following pattern:
* <pre>
* LongValues values = ..;
* final int numValues = values.setDocId(docId);
* for (int i = 0; i < numValues; i++) {
* long value = values.nextValue();
* // process value
* }
* </pre>
*
*/
public abstract class LongValues {
/**
* An empty {@link LongValues instance}
*/
public static final LongValues EMPTY = new Empty();
private final boolean multiValued;
protected int docId;
/**
* Creates a new {@link LongValues} instance
* @param multiValued <code>true</code> iff this instance is multivalued. Otherwise <code>false</code>.
*/
protected LongValues(boolean multiValued) {
this.multiValued = multiValued;
}
/**
* Is one of the documents in this field data values is multi valued?
*/
public final boolean isMultiValued() {
return multiValued;
}
/**
* Sets iteration to the specified docID and returns the number of
* values for this document ID,
* @param docId document ID
*
* @see #nextValue()
*/
public abstract int setDocument(int docId);
/**
* Returns the next value for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
* <p>
* If this instance returns ordered values the <tt>Nth</tt> value is strictly less than the <tt>N+1</tt> value with
* respect to the {@link AtomicFieldData.Order} returned from {@link #getOrder()}. If this instance returns
* <i>unordered</i> values {@link #getOrder()} must return {@link AtomicFieldData.Order#NONE}
* Note: the values returned are de-duplicated, only unique values are returned.
* </p>
*
* @return the next value for the current docID set to {@link #setDocument(int)}.
*/
public abstract long nextValue();
/**
* Returns the order the values are returned from {@link #nextValue()}.
* <p> Note: {@link LongValues} have {@link AtomicFieldData.Order#NUMERIC} by default.</p>
*/
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NUMERIC;
}
/**
* Ordinal based {@link LongValues}.
*/
public static abstract class WithOrdinals extends LongValues {
protected final BytesValues.WithOrdinals ordinals;
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.ordinals = ordinals;
}
/**
* Returns the value for the given ordinal.
* @param ord the ordinal to lookup.
* @return a long value associated with the given ordinal.
*/
public abstract long getValueByOrd(long ord);
@Override
public int setDocument(int docId) {
this.docId = docId;
return ordinals.setDocument(docId);
}
@Override
public long nextValue() {
return getValueByOrd(ordinals.nextOrd());
}
}
private static final class Empty extends LongValues {
public Empty() {
super(false);
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public long nextValue() {
throw new ElasticsearchIllegalStateException("Empty LongValues has no next value");
}
}
/** Wrap a {@link DoubleValues} instance. */
public static LongValues asLongValues(final DoubleValues values) {
return new LongValues(values.isMultiValued()) {
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public long nextValue() {
return (long) values.nextValue();
}
};
}
}

71
src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.geo.GeoPoint;
/**
* A stateful lightweight per document set of {@link GeoPoint} values.
* To iterate over values in a document use the following pattern:
* <pre>
* GeoPointValues values = ..;
* values.setDocId(docId);
* final int numValues = values.count();
* for (int i = 0; i < numValues; i++) {
* GeoPoint value = values.valueAt(i);
* // process value
* }
* </pre>
* The set of values associated with a document might contain duplicates and
* comes in a non-specified order.
*/
public abstract class MultiGeoPointValues {
/**
* Creates a new {@link MultiGeoPointValues} instance
*/
protected MultiGeoPointValues() {
}
/**
* Sets iteration to the specified docID.
* @param docId document ID
*
* @see #valueAt(int)
* @see #count()
*/
public abstract void setDocument(int docId);
/**
* Return the number of geo points the current document has.
*/
public abstract int count();
/**
* Return the <code>i-th</code> value associated with the current document.
* Behavior is undefined when <code>i</code> is undefined or greater than
* or equal to {@link #count()}.
*
* Note: the returned {@link GeoPoint} might be shared across invocations.
*
* @return the next value for the current docID set to {@link #setDocument(int)}.
*/
public abstract GeoPoint valueAt(int i);
}

120
src/main/java/org/elasticsearch/index/fielddata/MurmurHash3Values.java
View File

@ -1,120 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import com.carrotsearch.hppc.hash.MurmurHash3;
import org.apache.lucene.util.BytesRef;
/**
*
*/
public class MurmurHash3Values {
public static LongValues wrap(DoubleValues values) {
return new Double(values);
}
public static LongValues wrap(LongValues values) {
return new Long(values);
}
public static LongValues wrap(BytesValues values) {
return new Bytes(values);
}
private static class Long extends LongValues {
private final LongValues values;
public Long(LongValues values) {
super(values.isMultiValued());
this.values = values;
}
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public long nextValue() {
return MurmurHash3.hash(values.nextValue());
}
@Override
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NONE;
}
}
private static class Double extends LongValues {
private final DoubleValues values;
public Double(DoubleValues values) {
super(values.isMultiValued());
this.values = values;
}
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public long nextValue() {
return MurmurHash3.hash(java.lang.Double.doubleToLongBits(values.nextValue()));
}
@Override
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NONE;
}
}
private static class Bytes extends LongValues {
private final org.elasticsearch.common.hash.MurmurHash3.Hash128 hash = new org.elasticsearch.common.hash.MurmurHash3.Hash128();
private final BytesValues values;
public Bytes(BytesValues values) {
super(values.isMultiValued());
this.values = values;
}
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public long nextValue() {
final BytesRef next = values.nextValue();
org.elasticsearch.common.hash.MurmurHash3.hash128(next.bytes, next.offset, next.length, 0, hash);
return hash.h1;
}
@Override
public AtomicFieldData.Order getOrder() {
return AtomicFieldData.Order.NONE;
}
}
}

38
src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
/**
* A per-document numeric value.
*/
public abstract class NumericDoubleValues {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
protected NumericDoubleValues() {}
/**
* Returns the numeric value for the specified document ID. This must return
* <tt>0d</tt> if the given doc ID has no value.
* @param docID document ID to lookup
* @return numeric value
*/
public abstract double get(int docID);
}

1
src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java
View File

@ -22,6 +22,7 @@ import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.index.fielddata.plain.AbstractIndexFieldData;
import java.io.IOException;

115
src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
View File

@ -19,7 +19,10 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.*;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.unit.DistanceUnit;
@ -29,6 +32,7 @@ import org.elasticsearch.common.util.SlicedObjectList;
import org.joda.time.DateTimeZone;
import org.joda.time.MutableDateTime;
import java.util.Arrays;
import java.util.List;
/**
@ -37,10 +41,6 @@ import java.util.List;
*/
public abstract class ScriptDocValues {
public static final Longs EMPTY_LONGS = new Longs(LongValues.EMPTY);
public static final Doubles EMPTY_DOUBLES = new Doubles(DoubleValues.EMPTY);
public static final GeoPoints EMPTY_GEOPOINTS = new GeoPoints(GeoPointValues.EMPTY);
public static final Strings EMPTY_STRINGS = new Strings(BytesValues.EMPTY);
protected int docId;
protected boolean listLoaded = false;
@ -55,13 +55,12 @@ public abstract class ScriptDocValues {
public final static class Strings extends ScriptDocValues {
private final BytesValues values;
private final CharsRef spare = new CharsRef();
private final SortedBinaryDocValues values;
private SlicedObjectList<String> list;
public Strings(BytesValues values) {
public Strings(SortedBinaryDocValues values) {
this.values = values;
list = new SlicedObjectList<String>(values.isMultiValued() ? new String[10] : new String[1]) {
list = new SlicedObjectList<String>(new String[0]) {
@Override
public void grow(int newLength) {
@ -69,49 +68,48 @@ public abstract class ScriptDocValues {
if (values.length >= newLength) {
return;
}
final String[] current = values;
values = new String[ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(current, 0, values, 0, current.length);
values = Arrays.copyOf(values, ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
}
};
}
@Override
public boolean isEmpty() {
return values.setDocument(docId) == 0;
values.setDocument(docId);
return values.count() == 0;
}
public BytesValues getInternalValues() {
public SortedBinaryDocValues getInternalValues() {
return this.values;
}
public BytesRef getBytesValue() {
int numValues = values.setDocument(docId);
if (numValues == 0) {
values.setDocument(docId);
if (values.count() > 0) {
return values.valueAt(0);
} else {
return null;
}
return values.nextValue();
}
public String getValue() {
String value = null;
if (values.setDocument(docId) > 0) {
UnicodeUtil.UTF8toUTF16(values.nextValue(), spare);
value = spare.toString();
BytesRef value = getBytesValue();
if (value == null) {
return null;
} else {
return value.utf8ToString();
}
return value;
}
public List<String> getValues() {
if (!listLoaded) {
final int numValues = values.setDocument(docId);
values.setDocument(docId);
final int numValues = values.count();
list.offset = 0;
list.grow(numValues);
list.length = numValues;
for (int i = 0; i < numValues; i++) {
BytesRef next = values.nextValue();
UnicodeUtil.UTF8toUTF16(next, spare);
list.values[i] = spare.toString();
list.values[i] = values.valueAt(i).utf8ToString();
}
listLoaded = true;
}
@ -122,40 +120,43 @@ public abstract class ScriptDocValues {
public static class Longs extends ScriptDocValues {
private final LongValues values;
private final SortedNumericDocValues values;
private final MutableDateTime date = new MutableDateTime(0, DateTimeZone.UTC);
private final SlicedLongList list;
public Longs(LongValues values) {
public Longs(SortedNumericDocValues values) {
this.values = values;
this.list = new SlicedLongList(values.isMultiValued() ? 10 : 1);
this.list = new SlicedLongList(0);
}
public LongValues getInternalValues() {
public SortedNumericDocValues getInternalValues() {
return this.values;
}
@Override
public boolean isEmpty() {
return values.setDocument(docId) == 0;
values.setDocument(docId);
return values.count() == 0;
}
public long getValue() {
int numValues = values.setDocument(docId);
values.setDocument(docId);
int numValues = values.count();
if (numValues == 0) {
return 0l;
}
return values.nextValue();
return values.valueAt(0);
}
public List<Long> getValues() {
if (!listLoaded) {
final int numValues = values.setDocument(docId);
values.setDocument(docId);
final int numValues = values.count();
list.offset = 0;
list.grow(numValues);
list.length = numValues;
for (int i = 0; i < numValues; i++) {
list.values[i] = values.nextValue();
list.values[i] = values.valueAt(i);
}
listLoaded = true;
}
@ -171,41 +172,44 @@ public abstract class ScriptDocValues {
public static class Doubles extends ScriptDocValues {
private final DoubleValues values;
private final SortedNumericDoubleValues values;
private final SlicedDoubleList list;
public Doubles(DoubleValues values) {
public Doubles(SortedNumericDoubleValues values) {
this.values = values;
this.list = new SlicedDoubleList(values.isMultiValued() ? 10 : 1);
this.list = new SlicedDoubleList(0);
}
public DoubleValues getInternalValues() {
public SortedNumericDoubleValues getInternalValues() {
return this.values;
}
@Override
public boolean isEmpty() {
return values.setDocument(docId) == 0;
values.setDocument(docId);
return values.count() == 0;
}
public double getValue() {
int numValues = values.setDocument(docId);
values.setDocument(docId);
int numValues = values.count();
if (numValues == 0) {
return 0d;
}
return values.nextValue();
return values.valueAt(0);
}
public List<Double> getValues() {
if (!listLoaded) {
int numValues = values.setDocument(docId);
values.setDocument(docId);
int numValues = values.count();
list.offset = 0;
list.grow(numValues);
list.length = numValues;
for (int i = 0; i < numValues; i++) {
list.values[i] = values.nextValue();
list.values[i] = values.valueAt(i);
}
listLoaded = true;
}
@ -215,12 +219,12 @@ public abstract class ScriptDocValues {
public static class GeoPoints extends ScriptDocValues {
private final GeoPointValues values;
private final MultiGeoPointValues values;
private final SlicedObjectList<GeoPoint> list;
public GeoPoints(GeoPointValues values) {
public GeoPoints(MultiGeoPointValues values) {
this.values = values;
list = new SlicedObjectList<GeoPoint>(values.isMultiValued() ? new GeoPoint[10] : new GeoPoint[1]) {
list = new SlicedObjectList<GeoPoint>(new GeoPoint[0]) {
@Override
public void grow(int newLength) {
@ -228,24 +232,24 @@ public abstract class ScriptDocValues {
if (values.length >= newLength) {
return;
}
final GeoPoint[] current = values;
values = new GeoPoint[ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(current, 0, values, 0, current.length);
values = Arrays.copyOf(values, ArrayUtil.oversize(newLength, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
}
};
}
@Override
public boolean isEmpty() {
return values.setDocument(docId) == 0;
values.setDocument(docId);
return values.count() == 0;
}
public GeoPoint getValue() {
int numValues = values.setDocument(docId);
values.setDocument(docId);
int numValues = values.count();
if (numValues == 0) {
return null;
}
return values.nextValue();
return values.valueAt(0);
}
public double getLat() {
@ -276,12 +280,13 @@ public abstract class ScriptDocValues {
public List<GeoPoint> getValues() {
if (!listLoaded) {
int numValues = values.setDocument(docId);
values.setDocument(docId);
int numValues = values.count();
list.offset = 0;
list.grow(numValues);
list.length = numValues;
for (int i = 0; i < numValues; i++) {
GeoPoint next = values.nextValue();
GeoPoint next = values.valueAt(i);
GeoPoint point = list.values[i];
if (point == null) {
point = list.values[i] = new GeoPoint();

66
src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.Bits;
import org.elasticsearch.common.geo.GeoPoint;
final class SingletonMultiGeoPointValues extends MultiGeoPointValues {
private final GeoPointValues in;
private final Bits docsWithField;
private GeoPoint value;
private int count;
SingletonMultiGeoPointValues(GeoPointValues in, Bits docsWithField) {
this.in = in;
this.docsWithField = docsWithField;
}
@Override
public void setDocument(int docID) {
value = in.get(docID);
if (value.lat() == 0 && value.lon() == 0 && docsWithField != null && !docsWithField.get(docID)) {
count = 0;
} else {
count = 1;
}
}
@Override
public int count() {
return count;
}
@Override
public GeoPoint valueAt(int index) {
assert index == 0;
return value;
}
public GeoPointValues getGeoPointValues() {
return in;
}
public Bits getDocsWithField() {
return docsWithField;
}
}

68
src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Bits.MatchAllBits;
import org.apache.lucene.util.BytesRef;
final class SingletonSortedBinaryDocValues extends SortedBinaryDocValues {
private final BinaryDocValues in;
private final Bits docsWithField;
private BytesRef value;
private int count;
SingletonSortedBinaryDocValues(BinaryDocValues in, Bits docsWithField) {
this.in = in;
this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField;
}
@Override
public void setDocument(int docID) {
value = in.get(docID);
if (value.length == 0 && docsWithField != null && !docsWithField.get(docID)) {
count = 0;
} else {
count = 1;
}
}
@Override
public int count() {
return count;
}
@Override
public BytesRef valueAt(int index) {
assert index == 0;
return value;
}
public BinaryDocValues getBinaryDocValues() {
return in;
}
public Bits getDocsWithField() {
return docsWithField;
}
}

71
src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Bits.MatchAllBits;
/**
* Exposes multi-valued view over a single-valued instance.
* <p>
* This can be used if you want to have one multi-valued implementation
* that works for single or multi-valued types.
*/
final class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues {
private final NumericDoubleValues in;
private final Bits docsWithField;
private double value;
private int count;
public SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) {
this.in = in;
this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField;
}
/** Return the wrapped {@link NumericDoubleValues} */
public NumericDoubleValues getNumericDoubleValues() {
return in;
}
/** Return the wrapped {@link Bits} */
public Bits getDocsWithField() {
return docsWithField;
}
@Override
public void setDocument(int doc) {
value = in.get(doc);
if (docsWithField != null && value == 0 && docsWithField.get(doc) == false) {
count = 0;
} else {
count = 1;
}
}
@Override
public double valueAt(int index) {
return value;
}
@Override
public int count() {
return count;
}
}

48
src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
/**
* A list of per-document binary values, sorted
* according to {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
* There might be dups however.
*/
public abstract class SortedBinaryDocValues {
/**
* Positions to the specified document
*/
public abstract void setDocument(int docId);
/**
* Return the number of values of the current document.
*/
public abstract int count();
/**
* Retrieve the value for the current document at the specified index.
* An index ranges from {@code 0} to {@code count()-1}.
* Note that the returned {@link BytesRef} might be reused across invocations.
*/
public abstract BytesRef valueAt(int index);
}

50
src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedNumericDocValues;
/**
* Clone of {@link SortedNumericDocValues} for double values.
*/
public abstract class SortedNumericDoubleValues {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
protected SortedNumericDoubleValues() {}
/**
* Positions to the specified document
*/
public abstract void setDocument(int doc);
/**
* Retrieve the value for the current document at the specified index.
* An index ranges from {@code 0} to {@code count()-1}.
*/
public abstract double valueAt(int index);
/**
* Retrieves the count of values for the current document.
* This may be zero if a document has no values.
*/
public abstract int count();
}

82
src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.*;
import java.util.Arrays;
/**
* Base class for building {@link SortedBinaryDocValues} instances based on unsorted content.
*/
public abstract class SortingBinaryDocValues extends SortedBinaryDocValues {
protected int count;
protected BytesRef[] values;
private final Sorter sorter;
protected SortingBinaryDocValues() {
values = new BytesRef[] { new BytesRef() };
sorter = new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
ArrayUtil.swap(values, i, j);
}
@Override
protected int compare(int i, int j) {
return values[i].compareTo(values[j]);
}
};
}
/**
* Make sure the {@link #values} array can store at least {@link #count} entries.
*/
protected final void grow() {
if (values.length < count) {
final int oldLen = values.length;
final int newLen = ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
values = Arrays.copyOf(values, newLen);
for (int i = oldLen; i < newLen; ++i) {
values[i] = new BytesRef();
}
}
}
/**
* Sort values that are stored between offsets <code>0</code> and
* {@link #count} of {@link #values}.
*/
protected final void sort() {
sorter.sort(0, count);
}
@Override
public final int count() {
return count;
}
@Override
public final BytesRef valueAt(int index) {
return values[index];
}
}

78
src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.Sorter;
/**
* Base class for building {@link SortedNumericDocValues} instances based on unsorted content.
*/
public abstract class SortingNumericDocValues extends SortedNumericDocValues {
protected int count;
protected long[] values;
private final Sorter sorter;
protected SortingNumericDocValues() {
values = new long[1];
sorter = new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
final long tmp = values[i];
values[i] = values[j];
values[j] = tmp;
}
@Override
protected int compare(int i, int j) {
return Long.compare(values[i], values[j]);
}
};
}
/**
* Make sure the {@link #values} array can store at least {@link #count} entries.
*/
protected final void grow() {
values = ArrayUtil.grow(values, count);
}
/**
* Sort values that are stored between offsets <code>0</code> and
* {@link #count} of {@link #values}.
*/
protected final void sort() {
sorter.sort(0, count);
}
@Override
public final int count() {
return count;
}
@Override
public final long valueAt(int index) {
return values[index];
}
}

75
src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.Sorter;
/**
* Base class for building {@link SortedNumericDoubleValues} instances based on unsorted content.
*/
public abstract class SortingNumericDoubleValues extends SortedNumericDoubleValues {
protected int count;
protected double[] values;
private final Sorter sorter;
protected SortingNumericDoubleValues() {
values = new double[1];
sorter = new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
final double tmp = values[i];
values[i] = values[j];
values[j] = tmp;
}
@Override
protected int compare(int i, int j) {
return Double.compare(values[i], values[j]);
}
};
}
/**
* Make sure the {@link #values} array can store at least {@link #count} entries.
*/
protected final void grow() {
values = ArrayUtil.grow(values, count);
}
/**
* Sort values that are stored between offsets <code>0</code> and
* {@link #count} of {@link #values}.
*/
protected final void sort() {
sorter.sort(0, count);
}
public final int count() {
return count;
}
public final double valueAt(int index) {
return values[index];
}
}

5
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -61,8 +62,8 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexFieldData.WithOrdinals) {
return new BytesRefOrdValComparator((IndexFieldData.WithOrdinals<?>) indexFieldData, numHits, sortMode, missingBytes);
if (indexFieldData instanceof IndexOrdinalsFieldData) {
return new BytesRefOrdValComparator((IndexOrdinalsFieldData) indexFieldData, numHits, sortMode, missingBytes);
}
return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes);
}

123
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
View File

@ -20,10 +20,10 @@
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -47,7 +47,7 @@ import java.io.IOException;
*/
public final class BytesRefOrdValComparator extends NestedWrappableComparator<BytesRef> {
final IndexFieldData.WithOrdinals<?> indexFieldData;
final IndexOrdinalsFieldData indexFieldData;
final BytesRef missingValue;
/* Ords for each slot, times 4.
@ -73,7 +73,7 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
/* Current reader's doc ord/values.
@lucene.internal */
BytesValues.WithOrdinals termsIndex;
SortedDocValues termsIndex;
long missingOrd;
/* Bottom slot, or -1 if queue isn't full yet
@ -88,7 +88,7 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
BytesRef top;
long topOrd;
public BytesRefOrdValComparator(IndexFieldData.WithOrdinals<?> indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
public BytesRefOrdValComparator(IndexOrdinalsFieldData indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
this.indexFieldData = indexFieldData;
this.sortMode = sortMode;
this.missingValue = missingValue;
@ -141,13 +141,10 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
}
class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
final BytesValues.WithOrdinals termsIndex;
final SortedDocValues termsIndex;
public PerSegmentComparator(BytesValues.WithOrdinals termsIndex) {
public PerSegmentComparator(SortedDocValues termsIndex) {
this.termsIndex = termsIndex;
if (termsIndex.getMaxOrd() > Long.MAX_VALUE / 4) {
throw new IllegalStateException("Current terms index pretends it has more than " + (Long.MAX_VALUE / 4) + " ordinals, which is unsupported by this impl");
}
}
@Override
@ -188,22 +185,18 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
return val1.compareTo(val2);
}
protected long getOrd(int doc) {
return termsIndex.getOrd(doc);
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final long docOrd = getOrd(doc);
final long comparableOrd = docOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? missingOrd : docOrd << 2;
final long docOrd = termsIndex.getOrd(doc);
final long comparableOrd = docOrd < 0 ? missingOrd : docOrd << 2;
return Long.compare(bottomOrd, comparableOrd);
}
@Override
public int compareTop(int doc) throws IOException {
final long ord = getOrd(doc);
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
final long ord = termsIndex.getOrd(doc);
if (ord < 0) {
return compareTopMissing();
} else {
final long comparableOrd = ord << 2;
@ -229,17 +222,17 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
@Override
public void copy(int slot, int doc) {
final long ord = getOrd(doc);
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
final int ord = termsIndex.getOrd(doc);
if (ord < 0) {
ords[slot] = missingOrd;
values[slot] = missingValue;
} else {
assert ord >= 0;
ords[slot] = ord << 2;
ords[slot] = ((long) ord) << 2;
if (values[slot] == null || values[slot] == missingValue) {
values[slot] = new BytesRef();
}
values[slot].copyBytes(termsIndex.getValueByOrd(ord));
values[slot].copyBytes(termsIndex.lookupOrd(ord));
}
readerGen[slot] = currentReaderGen;
}
@ -253,30 +246,30 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
}
// for assertions
private boolean consistentInsertedOrd(BytesValues.WithOrdinals termsIndex, long ord, BytesRef value) {
final long previousOrd = ord >> 2;
final long nextOrd = previousOrd + 1;
final BytesRef previous = previousOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? null : termsIndex.getValueByOrd(previousOrd);
private boolean consistentInsertedOrd(SortedDocValues termsIndex, long ord, BytesRef value) {
final int previousOrd = (int) (ord >> 2);
final int nextOrd = previousOrd + 1;
final BytesRef previous = previousOrd < 0 ? null : termsIndex.lookupOrd(previousOrd);
if ((ord & 3) == 0) { // there was an existing ord with the inserted value
assert compareValues(previous, value) == 0;
} else {
assert compareValues(previous, value) < 0;
}
if (nextOrd < termsIndex.getMaxOrd()) {
final BytesRef next = termsIndex.getValueByOrd(nextOrd);
if (nextOrd < termsIndex.getValueCount()) {
final BytesRef next = termsIndex.lookupOrd(nextOrd);
assert compareValues(value, next) < 0;
}
return true;
}
// find where to insert an ord in the current terms index
private long ordInCurrentReader(BytesValues.WithOrdinals termsIndex, BytesRef value) {
private long ordInCurrentReader(SortedDocValues termsIndex, BytesRef value) {
final long ord;
if (value == null) {
ord = BytesValues.WithOrdinals.MISSING_ORDINAL << 2;
ord = -1 << 2;
} else {
final long docOrd = binarySearch(termsIndex, value);
if (docOrd >= BytesValues.WithOrdinals.MIN_ORDINAL) {
final long docOrd = termsIndex.lookupTerm(value);
if (docOrd >= 0) {
// value exists in the current segment
ord = docOrd << 2;
} else {
@ -291,20 +284,10 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
termsIndex = indexFieldData.load(context).getBytesValues();
termsIndex = sortMode.select(indexFieldData.load(context).getOrdinalsValues(), -1);
missingOrd = ordInCurrentReader(termsIndex, missingValue);
assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
FieldComparator<BytesRef> perSegComp = null;
if (termsIndex.isMultiValued()) {
perSegComp = new PerSegmentComparator(termsIndex) {
@Override
protected long getOrd(int doc) {
return getRelevantOrd(termsIndex, doc, sortMode);
}
};
} else {
perSegComp = new PerSegmentComparator(termsIndex);
}
FieldComparator<BytesRef> perSegComp = new PerSegmentComparator(termsIndex);
currentReaderGen++;
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
@ -353,56 +336,4 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
return values[slot];
}
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
return binarySearch(a, key, BytesValues.WithOrdinals.MIN_ORDINAL, a.getMaxOrd() - 1);
}
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key, long low, long high) {
assert low != BytesValues.WithOrdinals.MISSING_ORDINAL;
assert high == BytesValues.WithOrdinals.MISSING_ORDINAL || (a.getValueByOrd(high) == null | a.getValueByOrd(high) != null); // make sure we actually can get these values
assert low == high + 1 || a.getValueByOrd(low) == null | a.getValueByOrd(low) != null;
while (low <= high) {
long mid = (low + high) >>> 1;
BytesRef midVal = a.getValueByOrd(mid);
int cmp;
if (midVal != null) {
cmp = midVal.compareTo(key);
} else {
cmp = -1;
}
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid;
}
return -(low + 1);
}
static long getRelevantOrd(BytesValues.WithOrdinals readerOrds, int docId, MultiValueMode sortMode) {
int length = readerOrds.setDocument(docId);
long relevantVal = sortMode.startLong();
long result = BytesValues.WithOrdinals.MISSING_ORDINAL;
assert sortMode == MultiValueMode.MAX || sortMode == MultiValueMode.MIN;
for (int i = 0; i < length; i++) {
result = relevantVal = sortMode.apply(readerOrds.nextOrd(), relevantVal);
}
assert result >= BytesValues.WithOrdinals.MISSING_ORDINAL;
assert result < readerOrds.getMaxOrd();
return result;
// Enable this when the api can tell us that the ords per doc are ordered
/*if (reversed) {
IntArrayRef ref = readerOrds.getOrds(docId);
if (ref.isEmpty()) {
return 0;
} else {
return ref.values[ref.end - 1]; // last element is the highest value.
}
} else {
return readerOrds.getOrd(docId); // returns the lowest value
}*/
}
}

14
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
View File

@ -20,10 +20,11 @@
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -43,7 +44,7 @@ public final class BytesRefValComparator extends NestedWrappableComparator<Bytes
private final BytesRef[] values;
private BytesRef bottom;
private BytesRef top;
private BytesValues docTerms;
private BinaryDocValues docTerms;
BytesRefValComparator(IndexFieldData<?> indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
this.sortMode = sortMode;
@ -61,18 +62,18 @@ public final class BytesRefValComparator extends NestedWrappableComparator<Bytes
@Override
public int compareBottom(int doc) throws IOException {
BytesRef val2 = sortMode.getRelevantValue(docTerms, doc, missingValue);
BytesRef val2 = docTerms.get(doc);
return compareValues(bottom, val2);
}
@Override
public int compareTop(int doc) throws IOException {
return top.compareTo(sortMode.getRelevantValue(docTerms, doc, missingValue));
return top.compareTo(docTerms.get(doc));
}
@Override
public void copy(int slot, int doc) throws IOException {
BytesRef relevantValue = sortMode.getRelevantValue(docTerms, doc, missingValue);
BytesRef relevantValue = docTerms.get(doc);
if (relevantValue == missingValue) {
values[slot] = missingValue;
} else {
@ -85,7 +86,8 @@ public final class BytesRefValComparator extends NestedWrappableComparator<Bytes
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
docTerms = indexFieldData.load(context).getBytesValues();
final SortedBinaryDocValues docTerms = indexFieldData.load(context).getBytesValues();
this.docTerms = sortMode.select(docTerms, missingValue);
return this;
}

9
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java
View File

@ -26,13 +26,12 @@ import java.io.IOException;
/**
*/
public final class DoubleValuesComparator extends DoubleValuesComparatorBase<Double> {
public class DoubleValuesComparator extends DoubleValuesComparatorBase<Double> {
private final double[] values;
public DoubleValuesComparator(IndexNumericFieldData<?> indexFieldData, double missingValue, int numHits, MultiValueMode sortMode) {
public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
assert indexFieldData.getNumericType().requiredBits() <= 64;
this.values = new double[numHits];
}
@ -50,7 +49,7 @@ public final class DoubleValuesComparator extends DoubleValuesComparatorBase<Dou
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] = readerValues.get(doc);
}
@Override
@ -60,7 +59,7 @@ public final class DoubleValuesComparator extends DoubleValuesComparatorBase<Dou
@Override
public void add(int slot, int doc) {
values[slot] += sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] += readerValues.get(doc);
}
@Override

20
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java
View File

@ -20,21 +20,22 @@ package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
abstract class DoubleValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
protected final IndexNumericFieldData<?> indexFieldData;
protected final IndexNumericFieldData indexFieldData;
protected final double missingValue;
protected double bottom;
protected DoubleValues readerValues;
protected NumericDoubleValues readerValues;
protected final MultiValueMode sortMode;
public DoubleValuesComparatorBase(IndexNumericFieldData<?> indexFieldData, double missingValue, MultiValueMode sortMode) {
public DoubleValuesComparatorBase(IndexNumericFieldData indexFieldData, double missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
@ -42,18 +43,23 @@ abstract class DoubleValuesComparatorBase<T extends Number> extends NumberCompar
@Override
public final int compareBottom(int doc) throws IOException {
final double v2 = sortMode.getRelevantValue(readerValues, doc, missingValue);
final double v2 = readerValues.get(doc);
return compare(bottom, v2);
}
@Override
public int compareTop(int doc) throws IOException {
return compare(top.doubleValue(), sortMode.getRelevantValue(readerValues, doc, missingValue));
return compare(top.doubleValue(), readerValues.get(doc));
}
protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) {
SortedNumericDoubleValues readerValues = indexFieldData.load(context).getDoubleValues();
return sortMode.select(readerValues, missingValue);
}
@Override
public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
readerValues = indexFieldData.load(context).getDoubleValues();
this.readerValues = getNumericDoubleValues(context);
return this;
}

4
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
View File

@ -32,11 +32,11 @@ import java.io.IOException;
*/
public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData<?> indexFieldData;
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
public DoubleValuesComparatorSource(IndexNumericFieldData<?> indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;

6
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java
View File

@ -29,7 +29,7 @@ public final class FloatValuesComparator extends DoubleValuesComparatorBase<Floa
private final float[] values;
public FloatValuesComparator(IndexNumericFieldData<?> indexFieldData, float missingValue, int numHits, MultiValueMode sortMode) {
public FloatValuesComparator(IndexNumericFieldData indexFieldData, float missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
assert indexFieldData.getNumericType().requiredBits() <= 32;
this.values = new float[numHits];
@ -49,7 +49,7 @@ public final class FloatValuesComparator extends DoubleValuesComparatorBase<Floa
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = (float) sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] = (float) readerValues.get(doc);
}
@Override
@ -59,7 +59,7 @@ public final class FloatValuesComparator extends DoubleValuesComparatorBase<Floa
@Override
public void add(int slot, int doc) {
values[slot] += (float) sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] += (float) readerValues.get(doc);
}
@Override

4
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
View File

@ -31,11 +31,11 @@ import java.io.IOException;
*/
public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData<?> indexFieldData;
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
public FloatValuesComparatorSource(IndexNumericFieldData<?> indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;

190
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java
View File

@ -1,190 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
*/
public class GeoDistanceComparator extends NumberComparatorBase<Double> {
protected final IndexGeoPointFieldData<?> indexFieldData;
protected final double lat;
protected final double lon;
protected final DistanceUnit unit;
protected final GeoDistance geoDistance;
protected final GeoDistance.FixedSourceDistance fixedSourceDistance;
protected final MultiValueMode sortMode;
private static final Double MISSING_VALUE = Double.MAX_VALUE;
private final double[] values;
private double bottom;
private GeoDistanceValues geoDistanceValues;
public GeoDistanceComparator(int numHits, IndexGeoPointFieldData<?> indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance, MultiValueMode sortMode) {
this.values = new double[numHits];
this.indexFieldData = indexFieldData;
this.lat = lat;
this.lon = lon;
this.unit = unit;
this.geoDistance = geoDistance;
this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, unit);
this.sortMode = sortMode;
}
@Override
public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
GeoPointValues readerValues = indexFieldData.load(context).getGeoPointValues();
if (readerValues.isMultiValued()) {
geoDistanceValues = new MV(readerValues, fixedSourceDistance, sortMode);
} else {
geoDistanceValues = new SV(readerValues, fixedSourceDistance);
}
return this;
}
@Override
public int compare(int slot1, int slot2) {
return Double.compare(values[slot1], values[slot2]);
}
@Override
public int compareBottom(int doc) {
final double v2 = geoDistanceValues.computeDistance(doc);
return Double.compare(bottom, v2);
}
@Override
public int compareTop(int doc) throws IOException {
double docValue = geoDistanceValues.computeDistance(doc);
return Double.compare(top, docValue);
}
@Override
public void copy(int slot, int doc) {
values[slot] = geoDistanceValues.computeDistance(doc);
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public Double value(int slot) {
return values[slot];
}
@Override
public void add(int slot, int doc) {
values[slot] += geoDistanceValues.computeDistance(doc);
}
@Override
public void divide(int slot, int divisor) {
values[slot] /= divisor;
}
@Override
public void missing(int slot) {
values[slot] = MISSING_VALUE;
}
@Override
public int compareBottomMissing() {
return Double.compare(bottom, MISSING_VALUE);
}
@Override
public int compareTopMissing() {
return Double.compare(top, MISSING_VALUE);
}
// Computes the distance based on geo points.
// Due to this abstractions the geo distance comparator doesn't need to deal with whether fields have one
// or multiple geo points per document.
private static abstract class GeoDistanceValues {
protected final GeoPointValues readerValues;
protected final GeoDistance.FixedSourceDistance fixedSourceDistance;
protected GeoDistanceValues(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance) {
this.readerValues = readerValues;
this.fixedSourceDistance = fixedSourceDistance;
}
public abstract double computeDistance(int doc);
}
// Deals with one geo point per document
private static final class SV extends GeoDistanceValues {
SV(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance) {
super(readerValues, fixedSourceDistance);
}
@Override
public double computeDistance(int doc) {
int numValues = readerValues.setDocument(doc);
double result = MISSING_VALUE;
for (int i = 0; i < numValues; i++) {
GeoPoint geoPoint = readerValues.nextValue();
return fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon());
}
return MISSING_VALUE;
}
}
// Deals with more than one geo point per document
private static final class MV extends GeoDistanceValues {
private final MultiValueMode sortMode;
MV(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance, MultiValueMode sortMode) {
super(readerValues, fixedSourceDistance);
this.sortMode = sortMode;
}
@Override
public double computeDistance(int doc) {
final int length = readerValues.setDocument(doc);
double distance = sortMode.startDouble();
double result = MISSING_VALUE;
for (int i = 0; i < length; i++) {
GeoPoint point = readerValues.nextValue();
result = distance = sortMode.apply(distance, fixedSourceDistance.calculate(point.lat(), point.lon()));
}
return sortMode.reduce(result, length);
}
}
}

62
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java
View File

@ -1,62 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
*/
public class GeoDistanceComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexGeoPointFieldData<?> indexFieldData;
private final double lat;
private final double lon;
private final DistanceUnit unit;
private final GeoDistance geoDistance;
private final MultiValueMode sortMode;
public GeoDistanceComparatorSource(IndexGeoPointFieldData<?> indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.lat = lat;
this.lon = lon;
this.unit = unit;
this.geoDistance = geoDistance;
this.sortMode = sortMode;
}
@Override
public SortField.Type reducedType() {
return SortField.Type.DOUBLE;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert indexFieldData.getFieldNames().indexName().equals(fieldname);
return new GeoDistanceComparator(numHits, indexFieldData, lat, lon, unit, geoDistance, sortMode);
}
}

6
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java
View File

@ -29,7 +29,7 @@ public final class LongValuesComparator extends LongValuesComparatorBase<Long> {
private final long[] values;
public LongValuesComparator(IndexNumericFieldData<?> indexFieldData, long missingValue, int numHits, MultiValueMode sortMode) {
public LongValuesComparator(IndexNumericFieldData indexFieldData, long missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
this.values = new long[numHits];
assert indexFieldData.getNumericType().requiredBits() <= 64;
@ -48,7 +48,7 @@ public final class LongValuesComparator extends LongValuesComparatorBase<Long> {
}
public void copy(int slot, int doc) throws IOException {
values[slot] = sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] = readerValues.get(doc);
}
@Override
@ -58,7 +58,7 @@ public final class LongValuesComparator extends LongValuesComparatorBase<Long> {
@Override
public void add(int slot, int doc) {
values[slot] += sortMode.getRelevantValue(readerValues, doc, missingValue);
values[slot] += readerValues.get(doc);
}
@Override

16
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java
View File

@ -19,23 +19,24 @@
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.LongValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
abstract class LongValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
protected final IndexNumericFieldData<?> indexFieldData;
protected final IndexNumericFieldData indexFieldData;
protected final long missingValue;
protected long bottom;
protected LongValues readerValues;
protected NumericDocValues readerValues;
protected final MultiValueMode sortMode;
public LongValuesComparatorBase(IndexNumericFieldData<?> indexFieldData, long missingValue, MultiValueMode sortMode) {
public LongValuesComparatorBase(IndexNumericFieldData indexFieldData, long missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
@ -43,18 +44,19 @@ abstract class LongValuesComparatorBase<T extends Number> extends NumberComparat
@Override
public final int compareBottom(int doc) throws IOException {
long v2 = sortMode.getRelevantValue(readerValues, doc, missingValue);
long v2 = readerValues.get(doc);
return Long.compare(bottom, v2);
}
@Override
public int compareTop(int doc) throws IOException {
return Long.compare(top.longValue(), sortMode.getRelevantValue(readerValues, doc, missingValue));
return Long.compare(top.longValue(), readerValues.get(doc));
}
@Override
public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
readerValues = indexFieldData.load(context).getLongValues();
SortedNumericDocValues readerValues = indexFieldData.load(context).getLongValues();
this.readerValues = sortMode.select(readerValues, missingValue);
return this;
}

4
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
View File

@ -31,11 +31,11 @@ import java.io.IOException;
*/
public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData<?> indexFieldData;
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
public LongValuesComparatorSource(IndexNumericFieldData<?> indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;

43
src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
View File

@ -19,23 +19,24 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.XOrdinalMap;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds;
/**
* A {@link BytesValues.WithOrdinals} implementation that returns ordinals that are global.
* A {@link RandomAccessOrds} implementation that returns ordinals that are global.
*/
public class GlobalOrdinalMapping extends BytesValues.WithOrdinals {
public class GlobalOrdinalMapping extends AbstractRandomAccessOrds {
private final BytesValues.WithOrdinals values;
private final RandomAccessOrds values;
private final XOrdinalMap ordinalMap;
private final LongValues mapping;
private final BytesValues.WithOrdinals[] bytesValues;
private final RandomAccessOrds[] bytesValues;
GlobalOrdinalMapping(XOrdinalMap ordinalMap, BytesValues.WithOrdinals[] bytesValues, int segmentIndex) {
super(bytesValues[segmentIndex].isMultiValued());
GlobalOrdinalMapping(XOrdinalMap ordinalMap, RandomAccessOrds[] bytesValues, int segmentIndex) {
super();
this.values = bytesValues[segmentIndex];
this.bytesValues = bytesValues;
this.ordinalMap = ordinalMap;
@ -43,42 +44,34 @@ public class GlobalOrdinalMapping extends BytesValues.WithOrdinals {
}
@Override
public long getMaxOrd() {
public long getValueCount() {
return ordinalMap.getValueCount();
}
// NOTE: careful if we change the API here: unnecessary branch for < 0 here hurts a lot.
// so if we already know the count (from setDocument), its bad to do it redundantly.
public long getGlobalOrd(long segmentOrd) {
public final long getGlobalOrd(long segmentOrd) {
return mapping.get(segmentOrd);
}
@Override
public long getOrd(int docId) {
long v = values.getOrd(docId);
if (v < 0) {
return v;
} else {
return getGlobalOrd(v);
}
public long ordAt(int index) {
return getGlobalOrd(values.ordAt(index));
}
@Override
public long nextOrd() {
return getGlobalOrd(values.nextOrd());
public void doSetDocument(int docId) {
values.setDocument(docId);
}
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
public int cardinality() {
return values.cardinality();
}
@Override
public BytesRef getValueByOrd(long globalOrd) {
public BytesRef lookupOrd(long globalOrd) {
final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);
int readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd);
return bytesValues[readerIndex].getValueByOrd(segmentOrd);
return bytesValues[readerIndex].lookupOrd(segmentOrd);
}
}

43
src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
View File

@ -20,16 +20,51 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.XOrdinalMap;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import java.io.IOException;
/**
*/
public interface GlobalOrdinalsBuilder {
* Utility class to build global ordinals.
*/
public enum GlobalOrdinalsBuilder {
;
IndexFieldData.WithOrdinals build(IndexReader indexReader, IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService) throws IOException;
/**
* Build global ordinals for the provided {@link IndexReader}.
*/
public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, Settings settings, CircuitBreakerService breakerService, ESLogger logger) throws IOException {
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()];
final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()];
for (int i = 0; i < indexReader.leaves().size(); ++i) {
atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i));
subs[i] = atomicFD[i].getOrdinalsValues();
}
final XOrdinalMap ordinalMap = XOrdinalMap.build(null, subs, PackedInts.DEFAULT);
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
logger.debug(
"Global-ordinals[{}][{}] took {} ms",
indexFieldData.getFieldNames().fullName(),
ordinalMap.getValueCount(),
(System.currentTimeMillis() - startTime)
);
}
return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(),
indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes
);
}
}

16
src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
View File

@ -25,16 +25,17 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.MultiValueMode;
/**
* {@link IndexFieldData} base class for concrete global ordinals implementations.
*/
public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexFieldData.WithOrdinals, Accountable {
public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
private final FieldMapper.Names fieldNames;
private final FieldDataType fieldDataType;
@ -48,17 +49,17 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
}
@Override
public AtomicFieldData.WithOrdinals loadDirect(AtomicReaderContext context) throws Exception {
public AtomicOrdinalsFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}
@Override
public WithOrdinals loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
return this;
}
@Override
public WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
return this;
}
@ -72,11 +73,6 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
return fieldDataType;
}
@Override
public boolean valuesOrdered() {
return false;
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
throw new UnsupportedOperationException("no global ordinals sorting yet");

76
src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsBuilder.java
View File

@ -1,76 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.XOrdinalMap;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import java.io.IOException;
/**
*/
public class InternalGlobalOrdinalsBuilder extends AbstractIndexComponent implements GlobalOrdinalsBuilder {
public InternalGlobalOrdinalsBuilder(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
}
@Override
public IndexFieldData.WithOrdinals build(final IndexReader indexReader, IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService) throws IOException {
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
final AtomicFieldData.WithOrdinals<?>[] atomicFD = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
final TermsEnum[] subs = new TermsEnum[indexReader.leaves().size()];
final long[] weights = new long[subs.length];
for (int i = 0; i < indexReader.leaves().size(); ++i) {
atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i));
BytesValues.WithOrdinals v = atomicFD[i].getBytesValues();
subs[i] = v.getTermsEnum();
weights[i] = v.getMaxOrd();
}
final XOrdinalMap ordinalMap = XOrdinalMap.build(null, subs, weights, PackedInts.DEFAULT);
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
logger.debug(
"Global-ordinals[{}][{}] took {} ms",
indexFieldData.getFieldNames().fullName(),
ordinalMap.getValueCount(),
(System.currentTimeMillis() - startTime)
);
}
return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(),
indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes
);
}
}

31
src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
View File

@ -19,13 +19,13 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.XOrdinalMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
/**
@ -35,7 +35,7 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
private final Atomic[] atomicReaders;
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData.WithOrdinals[] segmentAfd, XOrdinalMap ordinalMap, long memorySizeInBytes) {
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicOrdinalsFieldData[] segmentAfd, XOrdinalMap ordinalMap, long memorySizeInBytes) {
super(index, settings, fieldNames, fieldDataType, memorySizeInBytes);
this.atomicReaders = new Atomic[segmentAfd.length];
for (int i = 0; i < segmentAfd.length; i++) {
@ -44,32 +44,32 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
}
@Override
public AtomicFieldData.WithOrdinals load(AtomicReaderContext context) {
public AtomicOrdinalsFieldData load(AtomicReaderContext context) {
return atomicReaders[context.ord];
}
private final class Atomic implements AtomicFieldData.WithOrdinals {
private final class Atomic extends AbstractAtomicOrdinalsFieldData {
private final WithOrdinals afd;
private final AtomicOrdinalsFieldData afd;
private final XOrdinalMap ordinalMap;
private final int segmentIndex;
private Atomic(WithOrdinals afd, XOrdinalMap ordinalMap, int segmentIndex) {
private Atomic(AtomicOrdinalsFieldData afd, XOrdinalMap ordinalMap, int segmentIndex) {
this.afd = afd;
this.ordinalMap = ordinalMap;
this.segmentIndex = segmentIndex;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
final BytesValues.WithOrdinals values = afd.getBytesValues();
if (values.getMaxOrd() == ordinalMap.getValueCount()) {
public RandomAccessOrds getOrdinalsValues() {
final RandomAccessOrds values = afd.getOrdinalsValues();
if (values.getValueCount() == ordinalMap.getValueCount()) {
// segment ordinals match global ordinals
return values;
}
final BytesValues.WithOrdinals[] bytesValues = new BytesValues.WithOrdinals[atomicReaders.length];
final RandomAccessOrds[] bytesValues = new RandomAccessOrds[atomicReaders.length];
for (int i = 0; i < bytesValues.length; i++) {
bytesValues[i] = atomicReaders[i].afd.getBytesValues();
bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
}
return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
}
@ -79,11 +79,6 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
return afd.ramBytesUsed();
}
@Override
public ScriptDocValues getScriptValues() {
throw new UnsupportedOperationException("Script values not supported on global ordinals");
}
@Override
public void close() {
}

93
src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
View File

@ -19,13 +19,15 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds;
/**
* {@link Ordinals} implementation which is efficient at storing field data ordinals for multi-valued or sparse fields.
@ -54,13 +56,13 @@ public class MultiOrdinals extends Ordinals {
}
private final boolean multiValued;
private final long maxOrd;
private final long valueCount;
private final MonotonicAppendingLongBuffer endOffsets;
private final AppendingPackedLongBuffer ords;
public MultiOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) {
multiValued = builder.getNumMultiValuesDocs() > 0;
maxOrd = builder.getMaxOrd();
valueCount = builder.getValueCount();
endOffsets = new MonotonicAppendingLongBuffer(OFFSET_INIT_PAGE_COUNT, OFFSETS_PAGE_SIZE, acceptableOverheadRatio);
ords = new AppendingPackedLongBuffer(OFFSET_INIT_PAGE_COUNT, OFFSETS_PAGE_SIZE, acceptableOverheadRatio);
long lastEndOffset = 0;
@ -83,61 +85,88 @@ public class MultiOrdinals extends Ordinals {
}
@Override
public WithOrdinals ordinals(ValuesHolder values) {
return new MultiDocs(this, values);
public RandomAccessOrds ordinals(ValuesHolder values) {
if (multiValued) {
return new MultiDocs(this, values);
} else {
return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values));
}
}
public static class MultiDocs extends BytesValues.WithOrdinals {
private static class SingleDocs extends SortedDocValues {
private final long maxOrd;
private final int valueCount;
private final MonotonicAppendingLongBuffer endOffsets;
private final AppendingPackedLongBuffer ords;
private long offset;
private long limit;
private final ValuesHolder values;
MultiDocs(MultiOrdinals ordinals, ValuesHolder values) {
super(ordinals.multiValued);
this.maxOrd = ordinals.maxOrd;
SingleDocs(MultiOrdinals ordinals, ValuesHolder values) {
this.valueCount = (int) ordinals.valueCount;
this.endOffsets = ordinals.endOffsets;
this.ords = ordinals.ords;
this.values = values;
}
@Override
public long getMaxOrd() {
return maxOrd;
public int getOrd(int docId) {
final long offset = docId != 0 ? endOffsets.get(docId - 1) : 0;
return (int) ords.get(offset);
}
@Override
public long getOrd(int docId) {
final long startOffset = docId > 0 ? endOffsets.get(docId - 1) : 0;
final long endOffset = endOffsets.get(docId);
if (startOffset == endOffset) {
return MISSING_ORDINAL; // ord for missing values
} else {
return ords.get(startOffset);
}
public BytesRef lookupOrd(int ord) {
return values.lookupOrd(ord);
}
@Override
public long nextOrd() {
assert offset < limit;
return ords.get(offset++);
public int getValueCount() {
return valueCount;
}
}
private static class MultiDocs extends AbstractRandomAccessOrds {
private final long valueCount;
private final MonotonicAppendingLongBuffer endOffsets;
private final AppendingPackedLongBuffer ords;
private long offset;
private int cardinality;
private final ValuesHolder values;
MultiDocs(MultiOrdinals ordinals, ValuesHolder values) {
this.valueCount = ordinals.valueCount;
this.endOffsets = ordinals.endOffsets;
this.ords = ordinals.ords;
this.values = values;
}
@Override
public int setDocument(int docId) {
final long startOffset = docId > 0 ? endOffsets.get(docId - 1) : 0;
public long getValueCount() {
return valueCount;
}
@Override
public void doSetDocument(int docId) {
final long startOffset = docId != 0 ? endOffsets.get(docId - 1) : 0;
final long endOffset = endOffsets.get(docId);
offset = startOffset;
limit = endOffset;
return (int) (endOffset - startOffset);
cardinality = (int) (endOffset - startOffset);
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.getValueByOrd(ord);
public int cardinality() {
return cardinality;
}
@Override
public long ordAt(int index) {
return ords.get(offset + index);
}
@Override
public BytesRef lookupOrd(long ord) {
return values.lookupOrd(ord);
}
}
}

10
src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
View File

@ -19,9 +19,9 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
/**
* A thread safe ordinals abstraction. Ordinals can only be positive integers.
@ -30,7 +30,7 @@ public abstract class Ordinals implements Accountable {
public static final ValuesHolder NO_VALUES = new ValuesHolder() {
@Override
public BytesRef getValueByOrd(long ord) {
public BytesRef lookupOrd(long ord) {
throw new UnsupportedOperationException();
}
};
@ -40,15 +40,15 @@ public abstract class Ordinals implements Accountable {
*/
public abstract long ramBytesUsed();
public abstract BytesValues.WithOrdinals ordinals(ValuesHolder values);
public abstract RandomAccessOrds ordinals(ValuesHolder values);
public final BytesValues.WithOrdinals ordinals() {
public final RandomAccessOrds ordinals() {
return ordinals(NO_VALUES);
}
public static interface ValuesHolder {
public abstract BytesRef getValueByOrd(long ord);
public abstract BytesRef lookupOrd(long ord);
}

7
src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.BytesValues;
import java.io.Closeable;
import java.io.IOException;
@ -260,7 +259,7 @@ public final class OrdinalsBuilder implements Closeable {
}
private final int maxDoc;
private long currentOrd = BytesValues.WithOrdinals.MIN_ORDINAL - 1;
private long currentOrd = -1;
private int numDocsWithValue = 0;
private int numMultiValuedDocs = 0;
private int totalNumOrds = 0;
@ -370,7 +369,7 @@ public final class OrdinalsBuilder implements Closeable {
/**
* Returns the number of distinct ordinals in this builder.
*/
public long getMaxOrd() {
public long getValueCount() {
return currentOrd + 1;
}
@ -396,7 +395,7 @@ public final class OrdinalsBuilder implements Closeable {
*/
public Ordinals build(Settings settings) {
final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FASTEST);
if (numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getMaxOrd(), acceptableOverheadRatio)) {
if (numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio)) {
// MultiOrdinals can be smaller than SinglePackedOrdinals for sparse fields
return new MultiOrdinals(this, acceptableOverheadRatio);
} else {

54
src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
View File

@ -19,11 +19,12 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
/**
*/
@ -31,73 +32,52 @@ public class SinglePackedOrdinals extends Ordinals {
// ordinals with value 0 indicates no value
private final PackedInts.Reader reader;
private final long maxOrd;
private long size = -1;
private final int valueCount;
public SinglePackedOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) {
assert builder.getNumMultiValuesDocs() == 0;
this.maxOrd = builder.getMaxOrd();
this.valueCount = (int) builder.getValueCount();
// We don't reuse the builder as-is because it might have been built with a higher overhead ratio
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(maxOrd), acceptableOverheadRatio);
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(valueCount), acceptableOverheadRatio);
PackedInts.copy(builder.getFirstOrdinals(), 0, reader, 0, builder.maxDoc(), 8 * 1024);
this.reader = reader;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed();
}
return size;
return RamUsageEstimator.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed();
}
@Override
public WithOrdinals ordinals(ValuesHolder values) {
return new Docs(this, values);
public RandomAccessOrds ordinals(ValuesHolder values) {
return (RandomAccessOrds) DocValues.singleton(new Docs(this, values));
}
private static class Docs extends BytesValues.WithOrdinals {
private static class Docs extends SortedDocValues {
private final long maxOrd;
private final int maxOrd;
private final PackedInts.Reader reader;
private final ValuesHolder values;
private long currentOrdinal;
public Docs(SinglePackedOrdinals parent, ValuesHolder values) {
super(false);
this.maxOrd = parent.maxOrd;
this.maxOrd = parent.valueCount;
this.reader = parent.reader;
this.values = values;
}
@Override
public long getMaxOrd() {
public int getValueCount() {
return maxOrd;
}
@Override
public long getOrd(int docId) {
return currentOrdinal = reader.get(docId) - 1;
public BytesRef lookupOrd(int ord) {
return values.lookupOrd(ord);
}
@Override
public long nextOrd() {
assert currentOrdinal >= MIN_ORDINAL;
return currentOrdinal;
}
@Override
public int setDocument(int docId) {
currentOrdinal = reader.get(docId) - 1;
// either this is > 1 or 0 - in any case it prevents a branch!
return 1 + (int) Math.min(currentOrdinal, 0);
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.getValueByOrd(ord);
public int getOrd(int docID) {
return (int) (reader.get(docID) - 1);
}
}
}

55
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.elasticsearch.index.fielddata.*;
/**
*/
abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData {
@Override
public final SortedBinaryDocValues getBytesValues() {
return FieldData.toString(getGeoPointValues());
}
public final ScriptDocValues.GeoPoints getScriptValues() {
return new ScriptDocValues.GeoPoints(getGeoPointValues());
}
public static AtomicGeoPointFieldData empty(final int maxDoc) {
return new AbstractAtomicGeoPointFieldData() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public void close() {
}
@Override
public MultiGeoPointValues getGeoPointValues() {
return FieldData.emptyMultiGeoPoints(maxDoc);
}
};
}
}

62
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
/**
*/
public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData {
@Override
public final ScriptDocValues getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public final SortedBinaryDocValues getBytesValues() {
return FieldData.toString(getOrdinalsValues());
}
public static AtomicOrdinalsFieldData empty() {
return new AbstractAtomicOrdinalsFieldData() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public void close() {
}
@Override
public RandomAccessOrds getOrdinalsValues() {
return (RandomAccessOrds) DocValues.emptySortedSet();
}
};
}
}

107
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import java.util.Set;
/**
*/
abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFieldData {
@Override
public final ScriptDocValues getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public final SortedBinaryDocValues getBytesValues() {
return new SortedBinaryDocValues() {
private final BytesRef[] terms = new BytesRef[2];
private int count;
@Override
public void setDocument(int docId) {
count = 0;
for (String type : types()) {
final SortedDocValues values = getOrdinalsValues(type);
final int ord = values.getOrd(docId);
if (ord >= 0) {
terms[count++] = values.lookupOrd(ord);
}
}
assert count <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2";
if (count > 1) {
int cmp = terms[0].compareTo(terms[1]);
if (cmp > 0) {
ArrayUtil.swap(terms, 0, 1);
} else if (cmp == 0) {
// If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field.
count = 1;
}
}
}
@Override
public int count() {
return count;
}
@Override
public BytesRef valueAt(int index) {
return terms[index];
}
};
}
public static AtomicParentChildFieldData empty() {
return new AbstractAtomicParentChildFieldData() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public void close() {
}
@Override
public SortedDocValues getOrdinalsValues(String type) {
return DocValues.emptySorted();
}
@Override
public Set<String> types() {
return ImmutableSet.of();
}
};
}
}

5
src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.fielddata;
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
@ -28,6 +28,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
@ -68,7 +69,7 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
}
@Override
public final FD load(AtomicReaderContext context) {
public FD load(AtomicReaderContext context) {
try {
FD fd = cache.load(context, this);
return fd;

46
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointIndexFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
View File

@ -28,41 +28,16 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<AtomicGeoPointFieldData<ScriptDocValues>> implements IndexGeoPointFieldData<AtomicGeoPointFieldData<ScriptDocValues>> {
protected static class Empty extends AtomicGeoPointFieldData<ScriptDocValues> {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public BytesValues getBytesValues() {
return BytesValues.EMPTY;
}
@Override
public GeoPointValues getGeoPointValues() {
return GeoPointValues.EMPTY;
}
@Override
public ScriptDocValues getScriptValues() {
return ScriptDocValues.EMPTY_GEOPOINTS;
}
@Override
public void close() {
// no-op
}
}
abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<AtomicGeoPointFieldData> implements IndexGeoPointFieldData {
protected static class GeoPointEnum {
@ -84,7 +59,7 @@ abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<Ato
UnicodeUtil.UTF8toUTF16(term, spare);
int commaIndex = -1;
for (int i = 0; i < spare.length; i++) {
if (spare.chars[spare.offset + i] == ',') { // safes a string creation
if (spare.chars[spare.offset + i] == ',') { // saves a string creation
commaIndex = i;
break;
}
@ -100,17 +75,10 @@ abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<Ato
}
public AbstractGeoPointIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
public AbstractIndexGeoPointFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
}
@Override
public boolean valuesOrdered() {
// because we might have single values? we can dynamically update a flag to reflect that
// based on the atomic field data loaded
return false;
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");

46
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
View File

@ -26,39 +26,35 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> extends AbstractIndexFieldData<FD> implements IndexFieldData.WithOrdinals<FD> {
public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData<AtomicOrdinalsFieldData> implements IndexOrdinalsFieldData {
protected Settings frequency;
protected Settings regex;
protected final GlobalOrdinalsBuilder globalOrdinalsBuilder;
protected final CircuitBreakerService breakerService;
protected AbstractBytesIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType,
IndexFieldDataCache cache, GlobalOrdinalsBuilder globalOrdinalsBuilder, CircuitBreakerService breakerService) {
protected AbstractIndexOrdinalsFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType,
IndexFieldDataCache cache, CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter");
frequency = groups.get("frequency");
regex = groups.get("regex");
this.globalOrdinalsBuilder = globalOrdinalsBuilder;
this.breakerService = breakerService;
}
@Override
public final boolean valuesOrdered() {
return true;
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
@ -66,7 +62,7 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
}
@Override
public WithOrdinals loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
@ -83,8 +79,8 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
}
@Override
public WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception {
return globalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService);
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
}
protected TermsEnum filter(Terms terms, AtomicReader reader) throws IOException {
@ -95,13 +91,13 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
if (iterator != null && frequency != null) {
iterator = FrequencyFilter.filter(iterator, terms, reader, frequency);
}
if (iterator != null && regex != null) {
iterator = RegexFilter.filter(iterator, terms, reader, regex);
}
return iterator;
}
private static final class FrequencyFilter extends FilteredTermsEnum {
private int minFreq;
@ -111,7 +107,7 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
this.minFreq = minFreq;
this.maxFreq = maxFreq;
}
public static TermsEnum filter(TermsEnum toFilter, Terms terms, AtomicReader reader, Settings settings) throws IOException {
int docCount = terms.getDocCount();
if (docCount == -1) {
@ -126,11 +122,11 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
assert minFreq < maxFreq;
return new FrequencyFilter(toFilter, minFreq, maxFreq);
}
return toFilter;
}
@Override
protected AcceptStatus accept(BytesRef arg0) throws IOException {
int docFreq = docFreq();
@ -140,12 +136,12 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
return AcceptStatus.NO;
}
}
private static final class RegexFilter extends FilteredTermsEnum {
private final Matcher matcher;
private final CharsRef spare = new CharsRef();
public RegexFilter(TermsEnum delegate, Matcher matcher) {
super(delegate, false);
this.matcher = matcher;
@ -154,11 +150,11 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
String pattern = regex.get("pattern");
if (pattern == null) {
return iterator;
}
}
Pattern p = Pattern.compile(pattern);
return new RegexFilter(iterator, p.matcher(""));
}
@Override
protected AcceptStatus accept(BytesRef arg0) throws IOException {
UnicodeUtil.UTF8toUTF16(arg0, spare);

72
src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.SortedNumericDocValues;
import org.elasticsearch.index.fielddata.*;
/**
* Specialization of {@link AtomicNumericFieldData} for floating-point numerics.
*/
abstract class AtomicDoubleFieldData implements AtomicNumericFieldData {
private final long ramBytesUsed;
AtomicDoubleFieldData(long ramBytesUsed) {
this.ramBytesUsed = ramBytesUsed;
}
@Override
public long ramBytesUsed() {
return ramBytesUsed;
}
@Override
public final ScriptDocValues getScriptValues() {
return new ScriptDocValues.Doubles(getDoubleValues());
}
@Override
public final SortedBinaryDocValues getBytesValues() {
return FieldData.toString(getDoubleValues());
}
@Override
public final SortedNumericDocValues getLongValues() {
return FieldData.castToLong(getDoubleValues());
}
public static AtomicNumericFieldData empty(final int maxDoc) {
return new AtomicDoubleFieldData(0) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return FieldData.emptySortedNumericDoubles(maxDoc);
}
};
}
@Override
public void close() {
}
}

72
src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.SortedNumericDocValues;
import org.elasticsearch.index.fielddata.*;
/**
* Specialization of {@link AtomicNumericFieldData} for integers.
*/
abstract class AtomicLongFieldData implements AtomicNumericFieldData {
private final long ramBytesUsed;
AtomicLongFieldData(long ramBytesUsed) {
this.ramBytesUsed = ramBytesUsed;
}
@Override
public long ramBytesUsed() {
return ramBytesUsed;
}
@Override
public final ScriptDocValues getScriptValues() {
return new ScriptDocValues.Longs(getLongValues());
}
@Override
public final SortedBinaryDocValues getBytesValues() {
return FieldData.toString(getLongValues());
}
@Override
public final SortedNumericDoubleValues getDoubleValues() {
return FieldData.castToDouble(getLongValues());
}
public static AtomicNumericFieldData empty(final int maxDoc) {
return new AtomicLongFieldData(0) {
@Override
public SortedNumericDocValues getLongValues() {
return FieldData.emptySortedNumeric(maxDoc);
}
};
}
@Override
public void close() {
}
}

35
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
View File

@ -23,17 +23,14 @@ import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
import java.io.IOException;
/** {@link AtomicFieldData} impl on top of Lucene's binary doc values. */
public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.Strings> {
public class BinaryDVAtomicFieldData implements AtomicFieldData {
private final AtomicReader reader;
private final String field;
@ -44,30 +41,11 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
}
@Override
public long ramBytesUsed() {
// TODO: Lucene doesn't expose it right now
return -1;
}
@Override
public BytesValues getBytesValues() {
public SortedBinaryDocValues getBytesValues() {
try {
final BinaryDocValues values = DocValues.getBinary(reader, field);
final Bits docsWithField = DocValues.getDocsWithField(reader, field);
return new BytesValues(false) {
int docId;
@Override
public int setDocument(int docId) {
this.docId = docId;
return docsWithField.get(docId) ? 1 : 0;
}
@Override
public BytesRef nextValue() {
return values.get(docId);
}
};
return FieldData.singleton(values, docsWithField);
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("Cannot load doc values", e);
}
@ -83,4 +61,9 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
// no-op
}
@Override
public long ramBytesUsed() {
return -1; // unknown
}
}

7
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
View File

@ -24,8 +24,8 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BinaryDVAtomicFieldData> {
@ -33,11 +33,6 @@ public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements I
super(index, fieldNames, fieldDataType);
}
@Override
public boolean valuesOrdered() {
return false;
}
@Override
public BinaryDVAtomicFieldData load(AtomicReaderContext context) {
return new BinaryDVAtomicFieldData(context.reader(), fieldNames.indexName());

149
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericAtomicFieldData.java
View File

@ -1,149 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.index.fielddata.AbstractAtomicNumericFieldData;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
import org.elasticsearch.index.fielddata.LongValues;
final class BinaryDVNumericAtomicFieldData extends AbstractAtomicNumericFieldData {
private final BinaryDocValues values;
private final NumericType numericType;
BinaryDVNumericAtomicFieldData(BinaryDocValues values, NumericType numericType) {
super(numericType.isFloatingPoint());
this.values = values;
this.numericType = numericType;
}
@Override
public LongValues getLongValues() {
if (numericType.isFloatingPoint()) {
return LongValues.asLongValues(getDoubleValues());
}
return new LongValues(true) {
BytesRef bytes;
final ByteArrayDataInput in = new ByteArrayDataInput();
long[] longs = new long[8];
int i = Integer.MAX_VALUE;
int valueCount = 0;
@Override
public int setDocument(int docId) {
bytes = values.get(docId);
in.reset(bytes.bytes, bytes.offset, bytes.length);
if (!in.eof()) {
// first value uses vLong on top of zig-zag encoding, then deltas are encoded using vLong
long previousValue = longs[0] = ByteUtils.zigZagDecode(ByteUtils.readVLong(in));
valueCount = 1;
while (!in.eof()) {
longs = ArrayUtil.grow(longs, valueCount + 1);
previousValue = longs[valueCount++] = previousValue + ByteUtils.readVLong(in);
}
} else {
valueCount = 0;
}
i = 0;
return valueCount;
}
@Override
public long nextValue() {
assert i < valueCount;
return longs[i++];
}
};
}
@Override
public DoubleValues getDoubleValues() {
if (!numericType.isFloatingPoint()) {
return DoubleValues.asDoubleValues(getLongValues());
}
switch (numericType) {
case FLOAT:
return new DoubleValues(true) {
BytesRef bytes;
int i = Integer.MAX_VALUE;
int valueCount = 0;
@Override
public int setDocument(int docId) {
bytes = values.get(docId);
assert bytes.length % 4 == 0;
i = 0;
return valueCount = bytes.length / 4;
}
@Override
public double nextValue() {
assert i < valueCount;
return ByteUtils.readFloatLE(bytes.bytes, bytes.offset + i++ * 4);
}
};
case DOUBLE:
return new DoubleValues(true) {
BytesRef bytes;
int i = Integer.MAX_VALUE;
int valueCount = 0;
@Override
public int setDocument(int docId) {
bytes = values.get(docId);
assert bytes.length % 8 == 0;
i = 0;
return valueCount = bytes.length / 8;
}
@Override
public double nextValue() {
assert i < valueCount;
return ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + i++ * 8);
}
};
default:
throw new AssertionError();
}
}
@Override
public long ramBytesUsed() {
return -1; // Lucene doesn't expose it
}
@Override
public void close() {
// no-op
}
}

146
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java
View File

@ -21,11 +21,20 @@ package org.elasticsearch.index.fielddata.plain;
import com.google.common.base.Preconditions;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
@ -34,7 +43,7 @@ import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData<BinaryDVNumericAtomicFieldData> {
public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData {
private final NumericType numericType;
@ -44,11 +53,6 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple
this.numericType = numericType;
}
@Override
public boolean valuesOrdered() {
return false;
}
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode) {
switch (numericType) {
case FLOAT:
@ -62,16 +66,42 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple
}
@Override
public BinaryDVNumericAtomicFieldData load(AtomicReaderContext context) {
public AtomicNumericFieldData load(AtomicReaderContext context) {
try {
return new BinaryDVNumericAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName()), numericType);
final BinaryDocValues values = DocValues.getBinary(context.reader(), fieldNames.indexName());
if (numericType.isFloatingPoint()) {
return new AtomicDoubleFieldData(-1) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
switch (numericType) {
case FLOAT:
return new BinaryAsSortedNumericFloatValues(values);
case DOUBLE:
return new BinaryAsSortedNumericDoubleValues(values);
default:
throw new ElasticsearchIllegalArgumentException("" + numericType);
}
}
};
} else {
return new AtomicLongFieldData(-1) {
@Override
public SortedNumericDocValues getLongValues() {
return new BinaryAsSortedNumericDocValues(values);
}
};
}
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("Cannot load doc values", e);
}
}
@Override
public BinaryDVNumericAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public AtomicNumericFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}
@ -80,4 +110,102 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple
return numericType;
}
private static class BinaryAsSortedNumericDocValues extends SortedNumericDocValues {
private final BinaryDocValues values;
private BytesRef bytes;
private final ByteArrayDataInput in = new ByteArrayDataInput();
private long[] longs = new long[1];
private int count = 0;
BinaryAsSortedNumericDocValues(BinaryDocValues values) {
this.values = values;
}
@Override
public void setDocument(int docId) {
bytes = values.get(docId);
in.reset(bytes.bytes, bytes.offset, bytes.length);
if (!in.eof()) {
// first value uses vLong on top of zig-zag encoding, then deltas are encoded using vLong
long previousValue = longs[0] = ByteUtils.zigZagDecode(ByteUtils.readVLong(in));
count = 1;
while (!in.eof()) {
longs = ArrayUtil.grow(longs, count + 1);
previousValue = longs[count++] = previousValue + ByteUtils.readVLong(in);
}
} else {
count = 0;
}
}
@Override
public int count() {
return count;
}
@Override
public long valueAt(int index) {
return longs[index];
}
}
private static class BinaryAsSortedNumericDoubleValues extends SortedNumericDoubleValues {
private final BinaryDocValues values;
private BytesRef bytes;
private int valueCount = 0;
BinaryAsSortedNumericDoubleValues(BinaryDocValues values) {
this.values = values;
}
@Override
public void setDocument(int docId) {
bytes = values.get(docId);
assert bytes.length % 8 == 0;
valueCount = bytes.length / 8;
}
@Override
public int count() {
return valueCount;
}
@Override
public double valueAt(int index) {
return ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + index * 8);
}
}
private static class BinaryAsSortedNumericFloatValues extends SortedNumericDoubleValues {
private final BinaryDocValues values;
private BytesRef bytes;
private int valueCount = 0;
BinaryAsSortedNumericFloatValues(BinaryDocValues values) {
this.values = values;
}
@Override
public void setDocument(int docId) {
bytes = values.get(docId);
assert bytes.length % 4 == 0;
valueCount = bytes.length / 4;
}
@Override
public int count() {
return valueCount;
}
@Override
public double valueAt(int index) {
return ByteUtils.readFloatLE(bytes.bytes, bytes.offset + index * 4);
}
}
}

53
src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java
View File

@ -21,12 +21,16 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues> {
import java.util.Arrays;
final class BytesBinaryDVAtomicFieldData implements AtomicFieldData {
private final BinaryDocValues values;
@ -41,32 +45,47 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocVal
}
@Override
public BytesValues getBytesValues() {
return new BytesValues(true) {
public SortedBinaryDocValues getBytesValues() {
return new SortedBinaryDocValues() {
BytesRef bytes;
final BytesRef scratch = new BytesRef();
int count;
BytesRef[] refs = new BytesRef[0];
final ByteArrayDataInput in = new ByteArrayDataInput();
@Override
public int setDocument(int docId) {
bytes = values.get(docId);
public void setDocument(int docId) {
final BytesRef bytes = values.get(docId);
in.reset(bytes.bytes, bytes.offset, bytes.length);
if (bytes.length == 0) {
return 0;
count = 0;
} else {
return in.readVInt();
count = in.readVInt();
if (count > refs.length) {
final int previousLength = refs.length;
refs = Arrays.copyOf(refs, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
for (int i = previousLength; i < refs.length; ++i) {
refs[i] = new BytesRef();
}
}
for (int i = 0; i < count; ++i) {
final int length = in.readVInt();
final BytesRef scratch = refs[i];
scratch.grow(length);
in.readBytes(scratch.bytes, 0, length);
scratch.length = length;
scratch.offset = 0;
}
}
}
@Override
public BytesRef nextValue() {
final int length = in.readVInt();
scratch.grow(length);
in.readBytes(scratch.bytes, 0, length);
scratch.length = length;
scratch.offset = 0;
return scratch;
public int count() {
return count;
}
@Override
public BytesRef valueAt(int index) {
return refs[index];
}
};

8
src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
View File

@ -29,7 +29,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
@ -44,11 +43,6 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme
super(index, fieldNames, fieldDataType);
}
@Override
public boolean valuesOrdered() {
return false;
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
throw new ElasticsearchIllegalArgumentException("can't sort on binary field");
@ -72,7 +66,7 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
// Ignore breaker
final Names fieldNames = mapper.names();
return new BytesBinaryDVIndexFieldData(index, fieldNames, mapper.fieldDataType());

146
src/main/java/org/elasticsearch/index/fielddata/plain/BytesValuesWithOrdinalsTermsEnum.java
View File

@ -1,146 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import java.io.IOException;
import java.util.Comparator;
/**
* A general {@link org.apache.lucene.index.TermsEnum} to iterate over terms from a {@link AtomicFieldData.WithOrdinals}
* instance.
*/
public class BytesValuesWithOrdinalsTermsEnum extends TermsEnum {
private final BytesValues.WithOrdinals bytesValues;
private final long maxOrd;
private long currentOrd = BytesValues.WithOrdinals.MISSING_ORDINAL;
private BytesRef currentTerm;
public BytesValuesWithOrdinalsTermsEnum(BytesValues.WithOrdinals bytesValues) {
this.bytesValues = bytesValues;
this.maxOrd = bytesValues.getMaxOrd();
}
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
long ord = binarySearch(bytesValues, text);
if (ord >= 0) {
currentOrd = ord;
currentTerm = bytesValues.getValueByOrd(currentOrd);
return SeekStatus.FOUND;
} else {
currentOrd = -ord - 1;
if (ord >= maxOrd) {
return SeekStatus.END;
} else {
currentTerm = bytesValues.getValueByOrd(currentOrd);
return SeekStatus.NOT_FOUND;
}
}
}
@Override
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord < bytesValues.getMaxOrd();
currentOrd = ord;
if (currentOrd == BytesValues.WithOrdinals.MISSING_ORDINAL) {
currentTerm = null;
} else {
currentTerm = bytesValues.getValueByOrd(currentOrd);
}
}
@Override
public BytesRef term() throws IOException {
return currentTerm;
}
@Override
public long ord() throws IOException {
return currentOrd;
}
@Override
public int docFreq() throws IOException {
throw new UnsupportedOperationException("docFreq not supported");
}
@Override
public long totalTermFreq() throws IOException {
return -1;
}
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException("docs not supported");
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException("docsAndPositions not supported");
}
@Override
public BytesRef next() throws IOException {
if (++currentOrd < maxOrd) {
return currentTerm = bytesValues.getValueByOrd(currentOrd);
} else {
return null;
}
}
@Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
final private static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
long low = 1;
long high = a.getMaxOrd();
while (low <= high) {
long mid = (low + high) >>> 1;
BytesRef midVal = a.getValueByOrd(mid);
int cmp;
if (midVal != null) {
cmp = midVal.compareTo(key);
} else {
cmp = -1;
}
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid;
}
return -(low + 1);
}
}

21
src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java
View File

@ -23,25 +23,27 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
/**
* A field data implementation that forbids loading and will throw an {@link org.elasticsearch.ElasticsearchIllegalStateException} if you try to load
* {@link AtomicFieldData} instances.
*/
public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicFieldData<?>> {
public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicFieldData> {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<AtomicFieldData<?>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
public IndexFieldData<AtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) {
// Ignore Circuit Breaker
return new DisabledIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache);
}
@ -52,12 +54,7 @@ public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicF
}
@Override
public boolean valuesOrdered() {
return false;
}
@Override
public AtomicFieldData<?> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
throw fail();
}

10
src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
View File

@ -22,13 +22,14 @@ package org.elasticsearch.index.fielddata.plain;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
@ -46,12 +47,14 @@ public abstract class DocValuesIndexFieldData {
protected final Index index;
protected final Names fieldNames;
protected final FieldDataType fieldDataType;
protected final ESLogger logger;
public DocValuesIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) {
super();
this.index = index;
this.fieldNames = fieldNames;
this.fieldDataType = fieldDataType;
this.logger = Loggers.getLogger(getClass());
}
public final Names getFieldNames() {
@ -88,8 +91,7 @@ public abstract class DocValuesIndexFieldData {
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService,
GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
// Ignore Circuit Breaker
final FieldMapper.Names fieldNames = mapper.names();
final Settings fdSettings = mapper.fieldDataType().getSettings();
@ -107,7 +109,7 @@ public abstract class DocValuesIndexFieldData {
} else if (numericType != null) {
return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, mapper.fieldDataType());
} else {
return new SortedSetDVBytesIndexFieldData(index, cache, indexSettings, fieldNames, globalOrdinalBuilder,breakerService, mapper.fieldDataType());
return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, mapper.fieldDataType());
}
}

287
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
View File

@ -1,287 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFieldData {
public static DoubleArrayAtomicFieldData empty() {
return new Empty();
}
protected long size = -1;
public DoubleArrayAtomicFieldData() {
super(true);
}
@Override
public void close() {
}
static class Empty extends DoubleArrayAtomicFieldData {
Empty() {
super();
}
@Override
public LongValues getLongValues() {
return LongValues.EMPTY;
}
@Override
public DoubleValues getDoubleValues() {
return DoubleValues.EMPTY;
}
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public BytesValues getBytesValues() {
return BytesValues.EMPTY;
}
@Override
public ScriptDocValues getScriptValues() {
return ScriptDocValues.EMPTY_DOUBLES;
}
}
public static class WithOrdinals extends DoubleArrayAtomicFieldData {
private final DoubleArray values;
private final Ordinals ordinals;
public WithOrdinals(DoubleArray values, Ordinals ordinals) {
super();
this.values = values;
this.ordinals = ordinals;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + values.ramBytesUsed() + ordinals.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, ordinals.ordinals());
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues.WithOrdinals {
private final DoubleArray values;
LongValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public final long getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return (long) values.get(ord);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues.WithOrdinals {
private final DoubleArray values;
DoubleValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public double getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
}
}
/**
* A single valued case, where not all values are "set", so we have a FixedBitSet that
* indicates which values have an actual value.
*/
public static class SingleFixedSet extends DoubleArrayAtomicFieldData {
private final DoubleArray values;
private final FixedBitSet set;
public SingleFixedSet(DoubleArray values, FixedBitSet set) {
super();
this.values = values;
this.set = set;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.ramBytesUsed() + RamUsageEstimator.sizeOf(set.getBits());
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, set);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, set);
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues {
private final DoubleArray values;
private final FixedBitSet set;
LongValues(DoubleArray values, FixedBitSet set) {
super(false);
this.values = values;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public long nextValue() {
return (long) values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final DoubleArray values;
private final FixedBitSet set;
DoubleValues(DoubleArray values, FixedBitSet set) {
super(false);
this.values = values;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends DoubleArrayAtomicFieldData {
private final DoubleArray values;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(DoubleArray values) {
super();
this.values = values;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values);
}
static final class LongValues extends DenseLongValues {
private final DoubleArray values;
LongValues(DoubleArray values) {
super(false);
this.values = values;
}
@Override
public long nextValue() {
return (long) values.get(docId);
}
}
static final class DoubleValues extends DenseDoubleValues {
private final DoubleArray values;
DoubleValues(DoubleArray values) {
super(false);
this.values = values;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
}

120
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
View File

@ -19,9 +19,7 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.*;
import org.apache.lucene.util.*;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
@ -30,7 +28,6 @@ import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -41,7 +38,7 @@ import org.elasticsearch.search.MultiValueMode;
/**
*/
public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArrayAtomicFieldData> implements IndexNumericFieldData<DoubleArrayAtomicFieldData> {
public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumericFieldData> implements IndexNumericFieldData {
private final CircuitBreakerService breakerService;
@ -49,7 +46,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
@Override
public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
return new DoubleArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService);
}
}
@ -66,22 +63,15 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
}
@Override
public boolean valuesOrdered() {
// because we might have single values? we can dynamically update a flag to reflect that
// based on the atomic field data loaded
return false;
}
public AtomicNumericFieldData loadDirect(AtomicReaderContext context) throws Exception {
@Override
public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
final AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
DoubleArrayAtomicFieldData data = null;
AtomicNumericFieldData data = null;
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
data = DoubleArrayAtomicFieldData.empty();
data = AtomicDoubleFieldData.empty(reader.maxDoc());
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
@ -99,10 +89,19 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
values.set(numTerms++, NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term)));
}
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new DoubleArrayAtomicFieldData.WithOrdinals(values, build);
final DoubleArray finalValues = values;
final Ordinals build = builder.build(fieldDataType.getSettings());
RandomAccessOrds ordinals = build.ordinals();
if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return withOrdinals(build, finalValues, reader.maxDoc());
}
};
} else {
final FixedBitSet set = builder.buildDocsWithValuesSet();
@ -111,25 +110,38 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
long uniqueValuesArraySize = values.ramBytesUsed();
long ordinalsSize = build.ramBytesUsed();
if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) {
data = new DoubleArrayAtomicFieldData.WithOrdinals(values, build);
final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
success = true;
return data;
return data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return withOrdinals(build, finalValues, reader.maxDoc());
}
};
}
int maxDoc = reader.maxDoc();
DoubleArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(maxDoc);
final DoubleArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(maxDoc);
for (int i = 0; i < maxDoc; i++) {
final long ordinal = ordinals.getOrd(i);
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
final long ordinal = ordinals.nextOrd();
if (ordinal != SortedSetDocValues.NO_MORE_ORDS) {
sValues.set(i, values.get(ordinal));
}
}
assert sValues.size() == maxDoc;
if (set == null) {
data = new DoubleArrayAtomicFieldData.Single(sValues);
} else {
data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, set);
}
final long ramBytesUsed = sValues.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed());
data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return singles(sValues, set);
}
};
success = true;
}
success = true;
return data;
@ -146,4 +158,50 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new DoubleValuesComparatorSource(this, missingValue, sortMode);
}
private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final DoubleArray values, int maxDoc) {
final RandomAccessOrds ords = ordinals.ordinals();
final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords);
if (singleOrds != null) {
final NumericDoubleValues singleValues = new NumericDoubleValues() {
@Override
public double get(int docID) {
final int ord = singleOrds.getOrd(docID);
if (ord >= 0) {
return values.get(singleOrds.getOrd(docID));
} else {
return 0;
}
}
};
return FieldData.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc));
} else {
return new SortedNumericDoubleValues() {
@Override
public double valueAt(int index) {
return values.get(ords.ordAt(index));
}
@Override
public void setDocument(int doc) {
ords.setDocument(doc);
}
@Override
public int count() {
return ords.cardinality();
}
};
}
}
private static SortedNumericDoubleValues singles(final DoubleArray values, FixedBitSet set) {
final NumericDoubleValues numValues = new NumericDoubleValues() {
@Override
public double get(int docID) {
return values.get(docID);
}
};
return FieldData.singleton(numValues, set);
}
}

19
src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
View File

@ -18,22 +18,21 @@
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
import org.apache.lucene.util.fst.Util;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import java.io.IOException;
/**
*/
public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public class FSTBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
// 0 ordinal in values means no value (its null)
protected final Ordinals ordinals;
@ -63,16 +62,10 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
public RandomAccessOrds getOrdinalsValues() {
return ordinals.ordinals(new ValuesHolder(fst));
}
@Override
public ScriptDocValues.Strings getScriptValues() {
assert fst != null;
return new ScriptDocValues.Strings(getBytesValues());
}
private static class ValuesHolder implements Ordinals.ValuesHolder {
private final FST<Long> fst;
@ -91,8 +84,8 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
}
@Override
public BytesRef getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
public BytesRef lookupOrd(long ord) {
assert ord != SortedSetDocValues.NO_MORE_ORDS;
in.setPosition(0);
fst.getFirstArc(firstArc);
try {

23
src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesIndexFieldData.java
View File

@ -28,7 +28,6 @@ import org.apache.lucene.util.fst.Util;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -38,37 +37,37 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
/**
*/
public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
public class FSTBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
private final CircuitBreakerService breakerService;
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
GlobalOrdinalsBuilder globalOrdinalBuilder) {
return new FSTBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
public IndexOrdinalsFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) {
return new FSTBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService);
}
}
FSTBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType,
IndexFieldDataCache cache, CircuitBreakerService breakerService, GlobalOrdinalsBuilder globalOrdinalsBuilder) {
super(index, indexSettings, fieldNames, fieldDataType, cache, globalOrdinalsBuilder, breakerService);
IndexFieldDataCache cache, CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache, breakerService);
this.breakerService = breakerService;
}
@Override
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicOrdinalsFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
FSTBytesAtomicFieldData data = null;
AtomicOrdinalsFieldData data = null;
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.ramBytesUsed());
return AtomicFieldData.WithOrdinals.EMPTY;
data = AbstractAtomicOrdinalsFieldData.empty();
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
org.apache.lucene.util.fst.Builder<Long> fstBuilder = new org.apache.lucene.util.fst.Builder<>(INPUT_TYPE.BYTE1, outputs);

286
src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
View File

@ -1,286 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.util.FloatArray;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFieldData {
public static FloatArrayAtomicFieldData empty() {
return new Empty();
}
protected long size = -1;
public FloatArrayAtomicFieldData() {
super(true);
}
@Override
public void close() {
}
static class Empty extends FloatArrayAtomicFieldData {
Empty() {
super();
}
@Override
public LongValues getLongValues() {
return LongValues.EMPTY;
}
@Override
public DoubleValues getDoubleValues() {
return DoubleValues.EMPTY;
}
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public BytesValues getBytesValues() {
return BytesValues.EMPTY;
}
@Override
public ScriptDocValues getScriptValues() {
return ScriptDocValues.EMPTY_DOUBLES;
}
}
public static class WithOrdinals extends FloatArrayAtomicFieldData {
private final Ordinals ordinals;
private final FloatArray values;
public WithOrdinals(FloatArray values, Ordinals ordinals) {
super();
this.values = values;
this.ordinals = ordinals;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + values.ramBytesUsed() + ordinals.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, ordinals.ordinals());
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues.WithOrdinals {
private final FloatArray values;
LongValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public long getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return (long) values.get(ord);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues.WithOrdinals {
private final FloatArray values;
DoubleValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public double getValueByOrd(long ord) {
return values.get(ord);
}
}
}
/**
* A single valued case, where not all values are "set", so we have a FixedBitSet that
* indicates which values have an actual value.
*/
public static class SingleFixedSet extends FloatArrayAtomicFieldData {
private final FloatArray values;
private final FixedBitSet set;
public SingleFixedSet(FloatArray values, FixedBitSet set) {
super();
this.values = values;
this.set = set;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.ramBytesUsed() + RamUsageEstimator.sizeOf(set.getBits());
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, set);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, set);
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues {
private final FloatArray values;
private final FixedBitSet set;
LongValues(FloatArray values, FixedBitSet set) {
super(false);
this.values = values;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public long nextValue() {
return (long) values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final FloatArray values;
private final FixedBitSet set;
DoubleValues(FloatArray values, FixedBitSet set) {
super(false);
this.values = values;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends FloatArrayAtomicFieldData {
private final FloatArray values;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(FloatArray values) {
super();
this.values = values;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values);
}
static class LongValues extends DenseLongValues {
private final FloatArray values;
LongValues(FloatArray values) {
super(false);
this.values = values;
}
@Override
public long nextValue() {
return (long) values.get(docId);
}
}
static class DoubleValues extends DenseDoubleValues {
private final FloatArray values;
DoubleValues(FloatArray values) {
super(false);
this.values = values;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
}

120
src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
View File

@ -18,9 +18,7 @@
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.*;
import org.apache.lucene.util.*;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
@ -29,7 +27,6 @@ import org.elasticsearch.common.util.FloatArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -40,7 +37,7 @@ import org.elasticsearch.search.MultiValueMode;
/**
*/
public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayAtomicFieldData> implements IndexNumericFieldData<FloatArrayAtomicFieldData> {
public class FloatArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumericFieldData> implements IndexNumericFieldData {
private final CircuitBreakerService breakerService;
@ -48,7 +45,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
@Override
public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
return new FloatArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService);
}
}
@ -65,21 +62,14 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
}
@Override
public boolean valuesOrdered() {
// because we might have single values? we can dynamically update a flag to reflect that
// based on the atomic field data loaded
return false;
}
@Override
public FloatArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
public AtomicNumericFieldData loadDirect(AtomicReaderContext context) throws Exception {
final AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
FloatArrayAtomicFieldData data = null;
AtomicNumericFieldData data = null;
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
data = FloatArrayAtomicFieldData.empty();
data = AtomicDoubleFieldData.empty(reader.maxDoc());
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
@ -97,10 +87,19 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
values.set(numTerms++, NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term)));
}
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new FloatArrayAtomicFieldData.WithOrdinals(values, build);
final FloatArray finalValues = values;
final Ordinals build = builder.build(fieldDataType.getSettings());
RandomAccessOrds ordinals = build.ordinals();
if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return withOrdinals(build, finalValues, reader.maxDoc());
}
};
} else {
final FixedBitSet set = builder.buildDocsWithValuesSet();
@ -109,25 +108,38 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
long uniqueValuesArraySize = values.ramBytesUsed();
long ordinalsSize = build.ramBytesUsed();
if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) {
data = new FloatArrayAtomicFieldData.WithOrdinals(values, build);
final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
success = true;
return data;
return data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return withOrdinals(build, finalValues, reader.maxDoc());
}
};
}
int maxDoc = reader.maxDoc();
FloatArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(maxDoc);
final FloatArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(maxDoc);
for (int i = 0; i < maxDoc; i++) {
final long ordinal = ordinals.getOrd(i);
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
final long ordinal = ordinals.nextOrd();
if (ordinal != SortedSetDocValues.NO_MORE_ORDS) {
sValues.set(i, values.get(ordinal));
}
}
assert sValues.size() == maxDoc;
if (set == null) {
data = new FloatArrayAtomicFieldData.Single(sValues);
} else {
data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, set);
}
final long ramBytesUsed = sValues.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed());
data = new AtomicDoubleFieldData(ramBytesUsed) {
@Override
public SortedNumericDoubleValues getDoubleValues() {
return singles(sValues, set);
}
};
success = true;
}
success = true;
return data;
@ -144,4 +156,50 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new FloatValuesComparatorSource(this, missingValue, sortMode);
}
private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final FloatArray values, int maxDoc) {
final RandomAccessOrds ords = ordinals.ordinals();
final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords);
if (singleOrds != null) {
final NumericDoubleValues singleValues = new NumericDoubleValues() {
@Override
public double get(int docID) {
final int ord = singleOrds.getOrd(docID);
if (ord >= 0) {
return values.get(singleOrds.getOrd(docID));
} else {
return 0;
}
}
};
return FieldData.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc));
} else {
return new SortedNumericDoubleValues() {
@Override
public double valueAt(int index) {
return values.get(ords.ordAt(index));
}
@Override
public void setDocument(int doc) {
ords.setDocument(doc);
}
@Override
public int count() {
return ords.cardinality();
}
};
}
}
private static SortedNumericDoubleValues singles(final FloatArray values, FixedBitSet set) {
final NumericDoubleValues numValues = new NumericDoubleValues() {
@Override
public double get(int docID) {
return values.get(docID);
}
};
return FieldData.singleton(numValues, set);
}
}

61
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVAtomicFieldData.java
View File

@ -20,14 +20,19 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
final class GeoPointBinaryDVAtomicFieldData extends AtomicGeoPointFieldData<ScriptDocValues> {
import java.util.Arrays;
final class GeoPointBinaryDVAtomicFieldData extends AbstractAtomicGeoPointFieldData {
private static final int COORDINATE_SIZE = 8; // number of bytes per coordinate
private static final int GEOPOINT_SIZE = COORDINATE_SIZE * 2; // lat + lon
private final BinaryDocValues values;
@ -41,39 +46,45 @@ final class GeoPointBinaryDVAtomicFieldData extends AtomicGeoPointFieldData<Scri
return -1; // not exposed by Lucene
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.GeoPoints(getGeoPointValues());
}
@Override
public void close() {
// no-op
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValues(true) {
public MultiGeoPointValues getGeoPointValues() {
return new MultiGeoPointValues() {
BytesRef bytes;
int i = Integer.MAX_VALUE;
int valueCount = 0;
final GeoPoint point = new GeoPoint();
int count;
GeoPoint[] points = new GeoPoint[0];
@Override
public int setDocument(int docId) {
bytes = values.get(docId);
assert bytes.length % 16 == 0;
i = 0;
return valueCount = (bytes.length >>> 4);
public void setDocument(int docId) {
final BytesRef bytes = values.get(docId);
assert bytes.length % GEOPOINT_SIZE == 0;
count = (bytes.length >>> 4);
if (count > points.length) {
final int previousLength = points.length;
points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
for (int i = previousLength; i < points.length; ++i) {
points[i] = new GeoPoint();
}
}
for (int i = 0; i < count; ++i) {
final double lat = ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + i * GEOPOINT_SIZE);
final double lon = ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + i * GEOPOINT_SIZE + COORDINATE_SIZE);
points[i].reset(lat, lon);
}
}
@Override
public GeoPoint nextValue() {
assert i < 2 * valueCount;
final double lat = ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + i++ * 8);
final double lon = ByteUtils.readDoubleLE(bytes.bytes, bytes.offset + i++ * 8);
return point.reset(lat, lon);
public int count() {
return count;
}
@Override
public GeoPoint valueAt(int index) {
return points[index];
}
};

14
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
@ -36,24 +35,19 @@ import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData<AtomicGeoPointFieldData<ScriptDocValues>> {
public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData {
public GeoPointBinaryDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) {
super(index, fieldNames, fieldDataType);
}
@Override
public boolean valuesOrdered() {
return false;
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
}
@Override
public AtomicGeoPointFieldData<ScriptDocValues> load(AtomicReaderContext context) {
public AtomicGeoPointFieldData load(AtomicReaderContext context) {
try {
return new GeoPointBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName()));
} catch (IOException e) {
@ -62,7 +56,7 @@ public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData impl
}
@Override
public AtomicGeoPointFieldData<ScriptDocValues> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicGeoPointFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}
@ -70,7 +64,7 @@ public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData impl
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
// Ignore breaker
final FieldMapper.Names fieldNames = mapper.names();
return new GeoPointBinaryDVIndexFieldData(index, fieldNames, mapper.fieldDataType());

195
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointCompressedAtomicFieldData.java
View File

@ -18,86 +18,89 @@
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PagedMutable;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
/**
* Field data atomic impl for geo points with lossy compression.
*/
public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFieldData<ScriptDocValues> {
protected long size = -1;
public abstract class GeoPointCompressedAtomicFieldData extends AbstractAtomicGeoPointFieldData {
@Override
public void close() {
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.GeoPoints(getGeoPointValues());
}
static class WithOrdinals extends GeoPointCompressedAtomicFieldData {
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final Ordinals ordinals;
private final int maxDoc;
public WithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, Ordinals ordinals) {
public WithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, Ordinals ordinals, int maxDoc) {
super();
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
this.ordinals = ordinals;
this.maxDoc = maxDoc;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed();
}
return size;
return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed();
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesWithOrdinals(encoding, lon, lat, ordinals.ordinals());
}
public MultiGeoPointValues getGeoPointValues() {
final RandomAccessOrds ords = ordinals.ordinals();
final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords);
if (singleOrds != null) {
final GeoPoint point = new GeoPoint();
final GeoPointValues values = new GeoPointValues() {
@Override
public GeoPoint get(int docID) {
final int ord = singleOrds.getOrd(docID);
if (ord >= 0) {
encoding.decode(lat.get(ord), lon.get(ord), point);
} else {
point.reset(0, 0);
}
return point;
}
};
return FieldData.singleton(values, DocValues.docsWithValue(singleOrds, maxDoc));
} else {
final GeoPoint point = new GeoPoint();
return new MultiGeoPointValues() {
public static class GeoPointValuesWithOrdinals extends GeoPointValues {
@Override
public GeoPoint valueAt(int index) {
final long ord = ords.ordAt(index);
encoding.decode(lat.get(ord), lon.get(ord), point);
return point;
}
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final BytesValues.WithOrdinals ordinals;
@Override
public void setDocument(int docId) {
ords.setDocument(docId);
}
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesWithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
this.ordinals = ordinals;
}
@Override
public GeoPoint nextValue() {
final long ord = ordinals.nextOrd();
return encoding.decode(lat.get(ord), lon.get(ord), scratch);
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return ordinals.setDocument(docId);
@Override
public int count() {
return ords.cardinality();
}
};
}
}
}
@ -105,13 +108,13 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
/**
* Assumes unset values are marked in bitset, and docId is used as the index to the value array.
*/
public static class SingleFixedSet extends GeoPointCompressedAtomicFieldData {
public static class Single extends GeoPointCompressedAtomicFieldData {
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final FixedBitSet set;
public SingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set) {
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set) {
super();
this.encoding = encoding;
this.lon = lon;
@ -121,100 +124,20 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + RamUsageEstimator.sizeOf(set.getBits());
}
return size;
return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed());
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesSingleFixedSet(encoding, lon, lat, set);
}
static class GeoPointValuesSingleFixedSet extends GeoPointValues {
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lat, lon;
private final FixedBitSet set;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesSingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set) {
super(false);
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public GeoPoint nextValue() {
return encoding.decode(lat.get(docId), lon.get(docId), scratch);
}
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends GeoPointCompressedAtomicFieldData {
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat) {
super();
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + (lon.ramBytesUsed() + lat.ramBytesUsed());
}
return size;
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesSingle(encoding, lon, lat);
}
static class GeoPointValuesSingle extends GeoPointValues {
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesSingle(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat) {
super(false);
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return 1;
}
@Override
public GeoPoint nextValue() {
return encoding.decode(lat.get(docId), lon.get(docId), scratch);
}
public MultiGeoPointValues getGeoPointValues() {
final GeoPoint point = new GeoPoint();
final GeoPointValues values = new GeoPointValues() {
@Override
public GeoPoint get(int docID) {
encoding.decode(lat.get(docID), lon.get(docID), point);
return point;
}
};
return FieldData.singleton(values, set);
}
}
}

33
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointCompressedIndexFieldData.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.packed.PackedInts;
@ -31,7 +32,6 @@ import org.elasticsearch.common.unit.DistanceUnit.Distance;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -42,7 +42,7 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
/**
*/
public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexFieldData {
public class GeoPointCompressedIndexFieldData extends AbstractIndexGeoPointFieldData {
private static final String PRECISION_KEY = "precision";
private static final Distance DEFAULT_PRECISION_VALUE = new Distance(1, DistanceUnit.CENTIMETERS);
@ -52,7 +52,7 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
@Override
public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
FieldDataType type = mapper.fieldDataType();
final String precisionAsString = type.getSettings().get(PRECISION_KEY);
final Distance precision;
@ -76,7 +76,7 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
}
@Override
public AtomicGeoPointFieldData<ScriptDocValues> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicGeoPointFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
@ -84,7 +84,7 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
data = new Empty();
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
@ -114,21 +114,22 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
}
Ordinals build = builder.build(fieldDataType.getSettings());
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
if (lat.size() != ordinals.getMaxOrd()) {
lat = lat.resize(ordinals.getMaxOrd());
lon = lon.resize(ordinals.getMaxOrd());
RandomAccessOrds ordinals = build.ordinals();
if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
if (lat.size() != ordinals.getValueCount()) {
lat = lat.resize(ordinals.getValueCount());
lon = lon.resize(ordinals.getValueCount());
}
data = new GeoPointCompressedAtomicFieldData.WithOrdinals(encoding, lon, lat, build);
data = new GeoPointCompressedAtomicFieldData.WithOrdinals(encoding, lon, lat, build, reader.maxDoc());
} else {
int maxDoc = reader.maxDoc();
PagedMutable sLat = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
PagedMutable sLon = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
final long missing = encoding.encodeCoordinate(0);
for (int i = 0; i < maxDoc; i++) {
final long nativeOrdinal = ordinals.getOrd(i);
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
final long nativeOrdinal = ordinals.nextOrd();
if (nativeOrdinal >= 0) {
sLat.set(i, lat.get(nativeOrdinal));
sLon.set(i, lon.get(nativeOrdinal));
} else {
@ -137,11 +138,7 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
}
}
FixedBitSet set = builder.buildDocsWithValuesSet();
if (set == null) {
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat);
} else {
data = new GeoPointCompressedAtomicFieldData.SingleFixedSet(encoding, sLon, sLat, set);
}
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat, set);
}
success = true;
return data;

188
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
View File

@ -18,80 +18,83 @@
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointFieldData<ScriptDocValues> {
protected long size = -1;
public abstract class GeoPointDoubleArrayAtomicFieldData extends AbstractAtomicGeoPointFieldData {
@Override
public void close() {
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.GeoPoints(getGeoPointValues());
}
static class WithOrdinals extends GeoPointDoubleArrayAtomicFieldData {
private final DoubleArray lon, lat;
private final Ordinals ordinals;
private final int maxDoc;
public WithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals ordinals) {
public WithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals ordinals, int maxDoc) {
super();
this.lon = lon;
this.lat = lat;
this.ordinals = ordinals;
this.maxDoc = maxDoc;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed();
}
return size;
return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed();
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesWithOrdinals(lon, lat, ordinals.ordinals());
}
public MultiGeoPointValues getGeoPointValues() {
final RandomAccessOrds ords = ordinals.ordinals();
final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords);
if (singleOrds != null) {
final GeoPoint point = new GeoPoint();
final GeoPointValues values = new GeoPointValues() {
@Override
public GeoPoint get(int docID) {
final int ord = singleOrds.getOrd(docID);
if (ord >= 0) {
point.reset(lat.get(ord), lon.get(ord));
}
return point;
}
};
return FieldData.singleton(values, DocValues.docsWithValue(singleOrds, maxDoc));
} else {
final GeoPoint point = new GeoPoint();
return new MultiGeoPointValues() {
public static class GeoPointValuesWithOrdinals extends GeoPointValues {
@Override
public GeoPoint valueAt(int index) {
final long ord = ords.ordAt(index);
point.reset(lat.get(ord), lon.get(ord));
return point;
}
private final DoubleArray lon, lat;
private final BytesValues.WithOrdinals ordinals;
@Override
public void setDocument(int docId) {
ords.setDocument(docId);
}
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesWithOrdinals(DoubleArray lon, DoubleArray lat, BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.lon = lon;
this.lat = lat;
this.ordinals = ordinals;
}
@Override
public GeoPoint nextValue() {
final long ord = ordinals.nextOrd();
return scratch.reset(lat.get(ord), lon.get(ord));
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return ordinals.setDocument(docId);
@Override
public int count() {
return ords.cardinality();
}
};
}
}
}
@ -99,13 +102,12 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
/**
* Assumes unset values are marked in bitset, and docId is used as the index to the value array.
*/
public static class SingleFixedSet extends GeoPointDoubleArrayAtomicFieldData {
public static class Single extends GeoPointDoubleArrayAtomicFieldData {
private final DoubleArray lon, lat;
private final FixedBitSet set;
public SingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set) {
super();
public Single(DoubleArray lon, DoubleArray lat, FixedBitSet set) {
this.lon = lon;
this.lat = lat;
this.set = set;
@ -113,97 +115,21 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + RamUsageEstimator.sizeOf(set.getBits());
}
return size;
return RamUsageEstimator.NUM_BYTES_INT/*size*/ + lon.ramBytesUsed() + lat.ramBytesUsed() + (set == null ? 0 : set.ramBytesUsed());
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesSingleFixedSet(lon, lat, set);
}
static class GeoPointValuesSingleFixedSet extends GeoPointValues {
private final DoubleArray lon;
private final DoubleArray lat;
private final FixedBitSet set;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesSingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set) {
super(false);
this.lon = lon;
this.lat = lat;
this.set = set;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return set.get(docId) ? 1 : 0;
}
@Override
public GeoPoint nextValue() {
return scratch.reset(lat.get(docId), lon.get(docId));
}
public MultiGeoPointValues getGeoPointValues() {
final GeoPoint point = new GeoPoint();
final GeoPointValues values = new GeoPointValues() {
@Override
public GeoPoint get(int docID) {
point.reset(lat.get(docID), lon.get(docID));
return point;
}
};
return FieldData.singleton(values, set);
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends GeoPointDoubleArrayAtomicFieldData {
private final DoubleArray lon, lat;
public Single(DoubleArray lon, DoubleArray lat) {
super();
this.lon = lon;
this.lat = lat;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + RamUsageEstimator.NUM_BYTES_INT/*numDocs*/ + (lon.ramBytesUsed() + lat.ramBytesUsed());
}
return size;
}
@Override
public GeoPointValues getGeoPointValues() {
return new GeoPointValuesSingle(lon, lat);
}
static class GeoPointValuesSingle extends GeoPointValues {
private final DoubleArray lon;
private final DoubleArray lat;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesSingle(DoubleArray lon, DoubleArray lat) {
super(false);
this.lon = lon;
this.lat = lat;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return 1;
}
@Override
public GeoPoint nextValue() {
return scratch.reset(lat.get(docId), lon.get(docId));
}
}
}
}

27
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.geo.GeoPoint;
@ -28,7 +29,6 @@ import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -38,7 +38,7 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
/**
*/
public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFieldData {
public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexGeoPointFieldData {
private final CircuitBreakerService breakerService;
@ -46,7 +46,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
@Override
public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
return new GeoPointDoubleArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService);
}
}
@ -58,7 +58,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
}
@Override
public AtomicGeoPointFieldData<ScriptDocValues> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicGeoPointFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
@ -66,7 +66,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
data = new Empty();
data = AbstractAtomicGeoPointFieldData.empty(reader.maxDoc());
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
@ -89,26 +89,23 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
BytesValues.WithOrdinals ordinals = build.ordinals();
if (!(ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
RandomAccessOrds ordinals = build.ordinals();
if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
int maxDoc = reader.maxDoc();
DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
for (int i = 0; i < maxDoc; i++) {
long nativeOrdinal = ordinals.getOrd(i);
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
long nativeOrdinal = ordinals.nextOrd();
if (nativeOrdinal != RandomAccessOrds.NO_MORE_ORDS) {
sLat.set(i, lat.get(nativeOrdinal));
sLon.set(i, lon.get(nativeOrdinal));
}
}
FixedBitSet set = builder.buildDocsWithValuesSet();
if (set == null) {
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat);
} else {
data = new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, set);
}
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, set);
} else {
data = new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon, lat, build);
data = new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon, lat, build, reader.maxDoc());
}
success = true;
return data;

126
src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
View File

@ -19,74 +19,52 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFieldData.WithOrdinals<ScriptDocValues>> {
public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache,
CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
CircuitBreakerService breakerService, MapperService mapperService) {
return new IndexIndexFieldData(index, mapper.names());
}
}
private static class IndexBytesValues extends BytesValues.WithOrdinals {
private static class IndexAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
private final BytesRef scratch;
protected IndexBytesValues(String index) {
super(false);
scratch = new BytesRef();
scratch.copyChars(index);
}
@Override
public int setDocument(int docId) {
return 1;
}
@Override
public long nextOrd() {
return BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
public long getOrd(int docId) {
return BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
public long getMaxOrd() {
return 1;
}
@Override
public BytesRef getValueByOrd(long ord) {
return scratch;
}
}
private static class IndexAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues> {
private final String index;
private final RandomAccessOrds values;
IndexAtomicFieldData(String index) {
this.index = index;
final BytesRef term = new BytesRef(index);
final SortedDocValues sortedValues = new SortedDocValues() {
@Override
public BytesRef lookupOrd(int ord) {
return term;
}
@Override
public int getValueCount() {
return 1;
}
@Override
public int getOrd(int docID) {
return 0;
}
};
values = (RandomAccessOrds) DocValues.singleton(sortedValues);
}
@Override
@ -95,13 +73,8 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new IndexBytesValues(index);
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
public RandomAccessOrds getOrdinalsValues() {
return values;
}
@Override
@ -110,38 +83,11 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
}
private final FieldMapper.Names names;
private final Index index;
private final AtomicOrdinalsFieldData atomicFieldData;
private IndexIndexFieldData(Index index, FieldMapper.Names names) {
this.index = index;
this.names = names;
}
@Override
public Index index() {
return index;
}
@Override
public org.elasticsearch.index.mapper.FieldMapper.Names getFieldNames() {
return names;
}
@Override
public FieldDataType getFieldDataType() {
return new FieldDataType("string");
}
@Override
public boolean valuesOrdered() {
return true;
}
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue,
MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
super(index, ImmutableSettings.EMPTY, names, new FieldDataType("string"), null, null);
atomicFieldData = new IndexAtomicFieldData(index().name());
}
@Override
@ -153,23 +99,23 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
}
@Override
public AtomicFieldData.WithOrdinals<ScriptDocValues> load(AtomicReaderContext context) {
return new IndexAtomicFieldData(index().name());
public final AtomicOrdinalsFieldData load(AtomicReaderContext context) {
return atomicFieldData;
}
@Override
public AtomicFieldData.WithOrdinals<ScriptDocValues> loadDirect(AtomicReaderContext context)
public AtomicOrdinalsFieldData loadDirect(AtomicReaderContext context)
throws Exception {
return load(context);
return atomicFieldData;
}
@Override
public IndexFieldData.WithOrdinals<?> loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
return this;
}
@Override
public IndexFieldData.WithOrdinals<?> localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
return loadGlobal(indexReader);
}

1
src/main/java/org/elasticsearch/index/fielddata/plain/NonEstimatingEstimator.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.index.fielddata.AbstractIndexFieldData;
import java.io.IOException;

116
src/main/java/org/elasticsearch/index/fielddata/plain/NumericDVAtomicFieldData.java
View File

@ -1,116 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.util.Bits;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.AbstractAtomicNumericFieldData;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.LongValues;
import java.io.IOException;
/** {@link AtomicFieldData} impl on top of Lucene's numeric doc values. */
public class NumericDVAtomicFieldData extends AbstractAtomicNumericFieldData {
private final AtomicReader reader;
private final String field;
public NumericDVAtomicFieldData(AtomicReader reader, String field) {
super(false);
this.reader = reader;
this.field = field;
}
@Override
public void close() {
// no-op
}
@Override
public long ramBytesUsed() {
// TODO: cannot be computed from Lucene
return -1;
}
private static class DocValuesAndBits {
final NumericDocValues values;
final Bits docsWithField;
DocValuesAndBits(NumericDocValues values, Bits docsWithField) {
super();
this.values = values;
this.docsWithField = docsWithField;
}
}
private DocValuesAndBits getDocValues() {
try {
final NumericDocValues values = DocValues.getNumeric(reader, field);
final Bits docsWithField = DocValues.getDocsWithField(reader, field);
return new DocValuesAndBits(values, docsWithField);
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("Cannot load doc values", e);
}
}
@Override
public LongValues getLongValues() {
final DocValuesAndBits docValues = getDocValues();
return new LongValues(false) {
@Override
public int setDocument(int docId) {
this.docId = docId;
return docValues.docsWithField.get(docId) ? 1 : 0;
}
@Override
public long nextValue() {
return docValues.values.get(docId);
}
};
}
@Override
public DoubleValues getDoubleValues() {
final DocValuesAndBits docValues = getDocValues();
return new DoubleValues(false) {
@Override
public int setDocument(int docId) {
this.docId = docId;
return docValues.docsWithField.get(docId) ? 1 : 0;
}
@Override
public double nextValue() {
return docValues.values.get(docId);
}
};
}
}

35
src/main/java/org/elasticsearch/index/fielddata/plain/NumericDVIndexFieldData.java
View File

@ -19,32 +19,45 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.*;
import org.apache.lucene.util.Bits;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
public class NumericDVIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData<NumericDVAtomicFieldData> {
import java.io.IOException;
public class NumericDVIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData {
public NumericDVIndexFieldData(Index index, Names fieldNames, FieldDataType fieldDataType) {
super(index, fieldNames, fieldDataType);
}
@Override
public boolean valuesOrdered() {
return false;
public AtomicLongFieldData load(AtomicReaderContext context) {
final AtomicReader reader = context.reader();
final String field = fieldNames.indexName();
return new AtomicLongFieldData(-1) {
@Override
public SortedNumericDocValues getLongValues() {
try {
final NumericDocValues values = DocValues.getNumeric(reader, field);
final Bits docsWithField = DocValues.getDocsWithField(reader, field);
return DocValues.singleton(values, docsWithField);
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("Cannot load doc values", e);
}
}
};
}
@Override
public NumericDVAtomicFieldData load(AtomicReaderContext context) {
return new NumericDVAtomicFieldData(context.reader(), fieldNames.indexName());
}
@Override
public NumericDVAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public AtomicLongFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}

465
src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
View File

@ -1,465 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.AppendingDeltaPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
* {@link AtomicNumericFieldData} implementation which stores data in packed arrays to save memory.
*/
public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFieldData {
public static PackedArrayAtomicFieldData empty() {
return new Empty();
}
protected long size = -1;
public PackedArrayAtomicFieldData() {
super(false);
}
@Override
public void close() {
}
static class Empty extends PackedArrayAtomicFieldData {
@Override
public LongValues getLongValues() {
return LongValues.EMPTY;
}
@Override
public DoubleValues getDoubleValues() {
return DoubleValues.EMPTY;
}
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public BytesValues getBytesValues() {
return BytesValues.EMPTY;
}
@Override
public ScriptDocValues getScriptValues() {
return ScriptDocValues.EMPTY_LONGS;
}
}
public static class WithOrdinals extends PackedArrayAtomicFieldData {
private final MonotonicAppendingLongBuffer values;
private final Ordinals ordinals;
public WithOrdinals(MonotonicAppendingLongBuffer values, Ordinals ordinals) {
super();
this.values = values;
this.ordinals = ordinals;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + values.ramBytesUsed() + ordinals.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, ordinals.ordinals());
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues.WithOrdinals {
private final MonotonicAppendingLongBuffer values;
LongValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public long getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues.WithOrdinals {
private final MonotonicAppendingLongBuffer values;
DoubleValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public double getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
}
}
/**
* A single valued case, where not all values are "set", so we have a special
* value which encodes the fact that the document has no value.
*/
public static class SingleSparse extends PackedArrayAtomicFieldData {
private final PackedInts.Mutable values;
private final long minValue;
private final long missingValue;
public SingleSparse(PackedInts.Mutable values, long minValue, long missingValue) {
super();
this.values = values;
this.minValue = minValue;
this.missingValue = missingValue;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = values.ramBytesUsed() + 2 * RamUsageEstimator.NUM_BYTES_LONG;
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, minValue, missingValue);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, minValue, missingValue);
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues {
private final PackedInts.Mutable values;
private final long minValue;
private final long missingValue;
LongValues(PackedInts.Mutable values, long minValue, long missingValue) {
super(false);
this.values = values;
this.minValue = minValue;
this.missingValue = missingValue;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return values.get(docId) != missingValue ? 1 : 0;
}
@Override
public long nextValue() {
return minValue + values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final PackedInts.Mutable values;
private final long minValue;
private final long missingValue;
DoubleValues(PackedInts.Mutable values, long minValue, long missingValue) {
super(false);
this.values = values;
this.minValue = minValue;
this.missingValue = missingValue;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return values.get(docId) != missingValue ? 1 : 0;
}
@Override
public double nextValue() {
return minValue + values.get(docId);
}
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends PackedArrayAtomicFieldData {
private final PackedInts.Mutable values;
private final long minValue;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(PackedInts.Mutable values, long minValue) {
super();
this.values = values;
this.minValue = minValue;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = values.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, minValue);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, minValue);
}
static class LongValues extends DenseLongValues {
private final PackedInts.Mutable values;
private final long minValue;
LongValues(PackedInts.Mutable values, long minValue) {
super(false);
this.values = values;
this.minValue = minValue;
}
@Override
public long nextValue() {
return minValue + values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final PackedInts.Mutable values;
private final long minValue;
DoubleValues(PackedInts.Mutable values, long minValue) {
super(false);
this.values = values;
this.minValue = minValue;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return 1;
}
@Override
public double nextValue() {
return minValue + values.get(docId);
}
}
}
/**
* A single valued case, where all values are "set" and are stored in a paged wise manner for better compression.
*/
public static class PagedSingle extends PackedArrayAtomicFieldData {
private final AppendingDeltaPackedLongBuffer values;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public PagedSingle(AppendingDeltaPackedLongBuffer values) {
super();
this.values = values;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = values.ramBytesUsed();
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values);
}
static class LongValues extends DenseLongValues {
private final AppendingDeltaPackedLongBuffer values;
LongValues(AppendingDeltaPackedLongBuffer values) {
super(false);
this.values = values;
}
@Override
public long nextValue() {
return values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final AppendingDeltaPackedLongBuffer values;
DoubleValues(AppendingDeltaPackedLongBuffer values) {
super(false);
this.values = values;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return 1;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
/**
* A single valued case, where not all values are "set", so we have a special
* value which encodes the fact that the document has no value. The data is stored in
* a paged wise manner for better compression.
*/
public static class PagedSingleSparse extends PackedArrayAtomicFieldData {
private final AppendingDeltaPackedLongBuffer values;
private final FixedBitSet docsWithValue;
public PagedSingleSparse(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue) {
super();
this.values = values;
this.docsWithValue = docsWithValue;
}
@Override
public long ramBytesUsed() {
if (size == -1) {
size = values.ramBytesUsed() + 2 * RamUsageEstimator.NUM_BYTES_LONG;
}
return size;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, docsWithValue);
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, docsWithValue);
}
static class LongValues extends org.elasticsearch.index.fielddata.LongValues {
private final AppendingDeltaPackedLongBuffer values;
private final FixedBitSet docsWithValue;
LongValues(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue) {
super(false);
this.values = values;
this.docsWithValue = docsWithValue;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return docsWithValue.get(docId) ? 1 : 0;
}
@Override
public long nextValue() {
return values.get(docId);
}
}
static class DoubleValues extends org.elasticsearch.index.fielddata.DoubleValues {
private final AppendingDeltaPackedLongBuffer values;
private final FixedBitSet docsWithValue;
DoubleValues(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue) {
super(false);
this.values = values;
this.docsWithValue = docsWithValue;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return docsWithValue.get(docId) ? 1 : 0;
}
@Override
public double nextValue() {
return values.get(docId);
}
}
}
}

219
src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java
View File

@ -20,10 +20,7 @@
package org.elasticsearch.index.fielddata.plain;
import com.google.common.base.Preconditions;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.*;
import org.apache.lucene.util.*;
import org.apache.lucene.util.packed.AppendingDeltaPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
@ -35,7 +32,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -50,7 +46,7 @@ import java.util.EnumSet;
/**
* Stores numeric data into bit-packed arrays for better memory efficiency.
*/
public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumericFieldData> implements IndexNumericFieldData<AtomicNumericFieldData> {
public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumericFieldData> implements IndexNumericFieldData {
public static class Builder implements IndexFieldData.Builder {
@ -63,7 +59,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
@Override
public IndexFieldData<AtomicNumericFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) {
return new PackedArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, numericType, breakerService);
}
}
@ -86,21 +82,14 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
return numericType;
}
@Override
public boolean valuesOrdered() {
// because we might have single values? we can dynamically update a flag to reflect that
// based on the atomic field data loaded
return false;
}
@Override
public AtomicNumericFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
final AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
PackedArrayAtomicFieldData data = null;
AtomicNumericFieldData data = null;
PackedArrayEstimator estimator = new PackedArrayEstimator(breakerService.getBreaker(), getNumericType(), getFieldNames().fullName());
if (terms == null) {
data = PackedArrayAtomicFieldData.empty();
data = AtomicLongFieldData.empty(reader.maxDoc());
estimator.adjustForNoTerms(data.ramBytesUsed());
return data;
}
@ -124,20 +113,28 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
assert values.size() == 0 || value > values.get(values.size() - 1);
values.add(value);
}
Ordinals build = builder.build(fieldDataType.getSettings());
final Ordinals build = builder.build(fieldDataType.getSettings());
CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType);
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new PackedArrayAtomicFieldData.WithOrdinals(values, build);
RandomAccessOrds ordinals = build.ordinals();
if (FieldData.isMultiValued(ordinals) || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
data = new AtomicLongFieldData(ramBytesUsed) {
@Override
public SortedNumericDocValues getLongValues() {
return withOrdinals(build, values, reader.maxDoc());
}
};
} else {
final FixedBitSet docsWithValues = builder.buildDocsWithValuesSet();
long minValue, maxValue;
minValue = maxValue = 0;
long minV, maxV;
minV = maxV = 0;
if (values.size() > 0) {
minValue = values.get(0);
maxValue = values.get(values.size() - 1);
minV = values.get(0);
maxV = values.get(values.size() - 1);
}
@ -145,7 +142,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
final int pageSize = fieldDataType.getSettings().getAsInt("single_value_page_size", 1024);
if (formatHint == null) {
formatHint = chooseStorageFormat(reader, values, build, ordinals, minValue, maxValue, acceptableOverheadRatio, pageSize);
formatHint = chooseStorageFormat(reader, values, build, ordinals, minV, maxV, acceptableOverheadRatio, pageSize);
}
logger.trace("single value format for field [{}] set to [{}]", getFieldNames().fullName(), formatHint);
@ -153,69 +150,93 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
switch (formatHint) {
case PACKED:
// Encode document without a value with a special value
long missingValue = 0;
long missingV = 0;
if (docsWithValues != null) {
if ((maxValue - minValue + 1) == values.size()) {
if ((maxV - minV + 1) == values.size()) {
// values are dense
if (minValue > Long.MIN_VALUE) {
missingValue = --minValue;
if (minV > Long.MIN_VALUE) {
missingV = --minV;
} else {
assert maxValue != Long.MAX_VALUE;
missingValue = ++maxValue;
assert maxV != Long.MAX_VALUE;
missingV = ++maxV;
}
} else {
for (long i = 1; i < values.size(); ++i) {
if (values.get(i) > values.get(i - 1) + 1) {
missingValue = values.get(i - 1) + 1;
missingV = values.get(i - 1) + 1;
break;
}
}
}
missingValue -= minValue;
missingV -= minV;
}
final long missingValue = missingV;
final long minValue = minV;
final long maxValue = maxV;
final long valuesDelta = maxValue - minValue;
int bitsRequired = valuesDelta < 0 ? 64 : PackedInts.bitsRequired(valuesDelta);
final PackedInts.Mutable sValues = PackedInts.getMutable(reader.maxDoc(), bitsRequired, acceptableOverheadRatio);
if (docsWithValues != null) {
sValues.fill(0, sValues.size(), missingValue);
sValues.fill(0, sValues.size(), missingV);
}
for (int i = 0; i < reader.maxDoc(); i++) {
final long ord = ordinals.getOrd(i);
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
if (ordinals.cardinality() > 0) {
final long ord = ordinals.ordAt(0);
long value = values.get(ord);
sValues.set(i, value - minValue);
}
}
if (docsWithValues == null) {
data = new PackedArrayAtomicFieldData.Single(sValues, minValue);
} else {
data = new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, missingValue);
}
long ramBytesUsed = values.ramBytesUsed() + (docsWithValues == null ? 0 : docsWithValues.ramBytesUsed());
data = new AtomicLongFieldData(ramBytesUsed) {
@Override
public SortedNumericDocValues getLongValues() {
if (docsWithValues == null) {
return singles(sValues, minValue);
} else {
return sparseSingles(sValues, minValue, missingValue, reader.maxDoc());
}
}
};
break;
case PAGED:
final AppendingDeltaPackedLongBuffer dpValues = new AppendingDeltaPackedLongBuffer(reader.maxDoc() / pageSize + 1, pageSize, acceptableOverheadRatio);
long lastValue = 0;
for (int i = 0; i < reader.maxDoc(); i++) {
final long ord = ordinals.getOrd(i);
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
if (ordinals.cardinality() > 0) {
final long ord = ordinals.ordAt(i);
lastValue = values.get(ord);
}
dpValues.add(lastValue);
}
dpValues.freeze();
if (docsWithValues == null) {
data = new PackedArrayAtomicFieldData.PagedSingle(dpValues);
} else {
data = new PackedArrayAtomicFieldData.PagedSingleSparse(dpValues, docsWithValues);
}
ramBytesUsed = dpValues.ramBytesUsed();
data = new AtomicLongFieldData(ramBytesUsed) {
@Override
public SortedNumericDocValues getLongValues() {
return pagedSingles(dpValues, docsWithValues);
}
};
break;
case ORDINALS:
data = new PackedArrayAtomicFieldData.WithOrdinals(values, build);
ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
data = new AtomicLongFieldData(ramBytesUsed) {
@Override
public SortedNumericDocValues getLongValues() {
return withOrdinals(build, values, reader.maxDoc());
}
};
break;
default:
throw new ElasticsearchException("unknown memory format: " + formatHint);
@ -238,7 +259,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
}
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, BytesValues.WithOrdinals ordinals,
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, RandomAccessOrds ordinals,
long minValue, long maxValue, float acceptableOverheadRatio, int pageSize) {
CommonSettings.MemoryStorageFormat format;
@ -259,8 +280,9 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
long pageMinOrdinal = Long.MAX_VALUE;
long pageMaxOrdinal = Long.MIN_VALUE;
for (int i = 1; i < reader.maxDoc(); ++i, pageIndex = (pageIndex + 1) % pageSize) {
long ordinal = ordinals.getOrd(i);
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
ordinals.setDocument(i);
if (ordinals.cardinality() > 0) {
long ordinal = ordinals.ordAt(0);
pageMaxOrdinal = Math.max(ordinal, pageMaxOrdinal);
pageMinOrdinal = Math.min(ordinal, pageMinOrdinal);
}
@ -385,4 +407,93 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
breaker.addWithoutBreaking(actualUsed);
}
}
private static SortedNumericDocValues withOrdinals(Ordinals ordinals, final LongValues values, int maxDoc) {
final RandomAccessOrds ords = ordinals.ordinals();
final SortedDocValues singleOrds = DocValues.unwrapSingleton(ords);
if (singleOrds != null) {
final NumericDocValues singleValues = new NumericDocValues() {
@Override
public long get(int docID) {
final int ord = singleOrds.getOrd(docID);
if (ord >= 0) {
return values.get(singleOrds.getOrd(docID));
} else {
return 0;
}
}
};
return DocValues.singleton(singleValues, DocValues.docsWithValue(ords, maxDoc));
} else {
return new SortedNumericDocValues() {
@Override
public long valueAt(int index) {
return values.get(ords.ordAt(index));
}
@Override
public void setDocument(int doc) {
ords.setDocument(doc);
}
@Override
public int count() {
return ords.cardinality();
}
};
}
}
private static SortedNumericDocValues singles(final NumericDocValues deltas, final long minValue) {
final NumericDocValues values;
if (minValue == 0) {
values = deltas;
} else {
values = new NumericDocValues() {
@Override
public long get(int docID) {
return minValue + deltas.get(docID);
}
};
}
return DocValues.singleton(values, null);
}
private static SortedNumericDocValues sparseSingles(final NumericDocValues deltas, final long minValue, final long missingValue, final int maxDoc) {
final NumericDocValues values = new NumericDocValues() {
@Override
public long get(int docID) {
final long delta = deltas.get(docID);
if (delta == missingValue) {
return 0;
}
return minValue + delta;
}
};
final Bits docsWithFields = new Bits() {
@Override
public boolean get(int index) {
return deltas.get(index) != missingValue;
}
@Override
public int length() {
return maxDoc;
}
};
return DocValues.singleton(values, docsWithFields);
}
private static SortedNumericDocValues pagedSingles(final AppendingDeltaPackedLongBuffer values, final FixedBitSet docsWithValue) {
return DocValues.singleton(new NumericDocValues() {
// we need to wrap since NumericDocValues must return 0 when a doc has no value
@Override
public long get(int docID) {
if (docsWithValue == null || docsWithValue.get(docID)) {
return values.get(docID);
} else {
return 0;
}
}
}, docsWithValue);
}
}

38
src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
View File

@ -18,30 +18,24 @@
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public class PagedBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
protected final Ordinals ordinals;
private long size = -1;
private final long readerBytesSize;
public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, long readerBytesSize, MonotonicAppendingLongBuffer termOrdToBytesOffset, Ordinals ordinals) {
public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, MonotonicAppendingLongBuffer termOrdToBytesOffset, Ordinals ordinals) {
this.bytes = bytes;
this.termOrdToBytesOffset = termOrdToBytesOffset;
this.ordinals = ordinals;
this.readerBytesSize = readerBytesSize;
}
@Override
@ -50,27 +44,19 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
@Override
public long ramBytesUsed() {
if (size == -1) {
long size = ordinals.ramBytesUsed();
// PackedBytes
size += readerBytesSize;
// PackedInts
size += termOrdToBytesOffset.ramBytesUsed();
this.size = size;
}
long size = ordinals.ramBytesUsed();
// PackedBytes
size += bytes.ramBytesUsed();
// PackedInts
size += termOrdToBytesOffset.ramBytesUsed();
return size;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
public RandomAccessOrds getOrdinalsValues() {
return ordinals.ordinals(new ValuesHolder(bytes, termOrdToBytesOffset));
}
@Override
public ScriptDocValues.Strings getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
private static class ValuesHolder implements Ordinals.ValuesHolder {
private final BytesRef scratch = new BytesRef();
@ -83,12 +69,12 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
}
@Override
public BytesRef getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
public BytesRef lookupOrd(long ord) {
assert ord >= 0;
bytes.fill(scratch, termOrdToBytesOffset.get(ord));
return scratch;
}
}
}

31
src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
View File

@ -28,7 +28,6 @@ import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
@ -40,34 +39,34 @@ import java.io.IOException;
/**
*/
public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
GlobalOrdinalsBuilder globalOrdinalBuilder) {
return new PagedBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
public IndexOrdinalsFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) {
return new PagedBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService);
}
}
public PagedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames,
FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService,
GlobalOrdinalsBuilder globalOrdinalsBuilder) {
super(index, indexSettings, fieldNames, fieldDataType, cache, globalOrdinalsBuilder, breakerService);
FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache, breakerService);
}
@Override
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
public AtomicOrdinalsFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
AtomicOrdinalsFieldData data = null;
PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(), getFieldNames().fullName());
Terms terms = reader.terms(getFieldNames().indexName());
if (terms == null) {
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.ramBytesUsed());
return AtomicFieldData.WithOrdinals.EMPTY;
data = AbstractAtomicOrdinalsFieldData.empty();
estimator.afterLoad(null, data.ramBytesUsed());
return data;
}
final PagedBytes bytes = new PagedBytes(15);
@ -85,14 +84,11 @@ public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<Atomic
// Wrap the context in an estimator and use it to either estimate
// the entire set, or wrap the TermsEnum so it can be calculated
// per-term
PagedBytesAtomicFieldData data = null;
TermsEnum termsEnum = estimator.beforeLoad(terms);
boolean success = false;
try (OrdinalsBuilder builder = new OrdinalsBuilder(numTerms, reader.maxDoc(), acceptableTransientOverheadRatio)) {
// 0 is reserved for "unset"
bytes.copyUsingLengthPrefix(new BytesRef());
DocsEnum docsEnum = null;
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
final long termOrd = builder.nextOrdinal();
@ -103,11 +99,10 @@ public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<Atomic
builder.addDoc(docId);
}
}
final long sizePointer = bytes.getPointer();
PagedBytes.Reader bytesReader = bytes.freeze(true);
final Ordinals ordinals = builder.build(fieldDataType.getSettings());
data = new PagedBytesAtomicFieldData(bytesReader, sizePointer, termOrdToBytesOffset, ordinals);
data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset, ordinals);
success = true;
return data;
} finally {

88
src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildAtomicFieldData.java
View File

@ -20,23 +20,26 @@
package org.elasticsearch.index.fielddata.plain;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.util.HashSet;
import java.util.Set;
/**
*/
public class ParentChildAtomicFieldData implements AtomicFieldData {
public class ParentChildAtomicFieldData extends AbstractAtomicParentChildFieldData {
private final ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds;
private final ImmutableOpenMap<String, AtomicOrdinalsFieldData> typeToIds;
private final long memorySizeInBytes;
public ParentChildAtomicFieldData(ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds) {
public ParentChildAtomicFieldData(ImmutableOpenMap<String, AtomicOrdinalsFieldData> typeToIds) {
this.typeToIds = typeToIds;
long size = 0;
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
for (ObjectCursor<AtomicOrdinalsFieldData> cursor : typeToIds.values()) {
size += cursor.value.ramBytesUsed();
}
this.memorySizeInBytes = size;
@ -48,76 +51,31 @@ public class ParentChildAtomicFieldData implements AtomicFieldData {
}
@Override
public BytesValues getBytesValues() {
final BytesValues[] bytesValues = new BytesValues[typeToIds.size()];
int index = 0;
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
bytesValues[index++] = cursor.value.getBytesValues();
public Set<String> types() {
final Set<String> types = new HashSet<>();
for (ObjectCursor<String> cursor : typeToIds.keys()) {
types.add(cursor.value);
}
return new BytesValues(true) {
private final BytesRef scratch = new BytesRef();
private final BytesRef[] terms = new BytesRef[2];
private int index;
@Override
public int setDocument(int docId) {
index = 0;
int counter = 0;
for (final BytesValues values : bytesValues) {
int numValues = values.setDocument(docId);
assert numValues <= 1 : "Per doc/type combination only a single value is allowed";
if (numValues == 1) {
terms[counter++] = BytesRef.deepCopyOf(values.nextValue());
}
}
assert counter <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2";
if (counter > 1) {
int cmp = terms[0].compareTo(terms[1]);
if (cmp > 0) {
BytesRef temp = terms[0];
terms[0] = terms[1];
terms[1] = temp;
} else if (cmp == 0) {
// If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field.
return 1;
}
}
return counter;
}
@Override
public BytesRef nextValue() {
BytesRef current = terms[index++];
scratch.bytes = current.bytes;
scratch.offset = current.offset;
scratch.length = current.length;
return scratch;
}
};
return types;
}
public BytesValues.WithOrdinals getBytesValues(String type) {
WithOrdinals atomicFieldData = typeToIds.get(type);
@Override
public SortedDocValues getOrdinalsValues(String type) {
AtomicOrdinalsFieldData atomicFieldData = typeToIds.get(type);
if (atomicFieldData != null) {
return atomicFieldData.getBytesValues();
return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues(), -1);
} else {
return null;
return DocValues.emptySorted();
}
}
public WithOrdinals getAtomicFieldData(String type) {
public AtomicOrdinalsFieldData getAtomicFieldData(String type) {
return typeToIds.get(type);
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public void close() {
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
for (ObjectCursor<AtomicOrdinalsFieldData> cursor : typeToIds.values()) {
cursor.value.close();
}
}

387
src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View File

@ -21,27 +21,28 @@ package org.elasticsearch.index.fielddata.plain;
import com.carrotsearch.hppc.ObjectObjectOpenHashMap;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableSortedSet;
import org.apache.lucene.index.*;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentTypeListener;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
@ -49,18 +50,17 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.TreeSet;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
* ParentChildIndexFieldData is responsible for loading the id cache mapping
* needed for has_child and has_parent queries into memory.
*/
public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChildAtomicFieldData> implements DocumentTypeListener {
public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicParentChildFieldData> implements IndexParentChildFieldData, DocumentTypeListener {
private final NavigableSet<BytesRef> parentTypes;
private final CircuitBreakerService breakerService;
private final GlobalOrdinalsBuilder globalOrdinalsBuilder;
// If child type (a type with _parent field) is added or removed, we want to make sure modifications don't happen
// while loading.
@ -68,22 +68,16 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
public ParentChildIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames,
FieldDataType fieldDataType, IndexFieldDataCache cache, MapperService mapperService,
CircuitBreakerService breakerService, GlobalOrdinalsBuilder globalOrdinalsBuilder) {
CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
parentTypes = new TreeSet<>(BytesRef.getUTF8SortedAsUnicodeComparator());
this.breakerService = breakerService;
this.globalOrdinalsBuilder = globalOrdinalsBuilder;
for (DocumentMapper documentMapper : mapperService.docMappers(false)) {
beforeCreate(documentMapper);
}
mapperService.addTypeListener(this);
}
@Override
public boolean valuesOrdered() {
return true;
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
@ -96,76 +90,71 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
"acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO
);
final NavigableSet<BytesRef> parentTypes;
synchronized (lock) {
boolean success = false;
ParentChildAtomicFieldData data = null;
ParentChildFilteredTermsEnum termsEnum = new ParentChildFilteredTermsEnum(
new ParentChildIntersectTermsEnum(reader, UidFieldMapper.NAME, ParentFieldMapper.NAME),
parentTypes
);
ParentChildEstimator estimator = new ParentChildEstimator(breakerService.getBreaker(), termsEnum);
TermsEnum estimatedTermsEnum = estimator.beforeLoad(null);
ObjectObjectOpenHashMap<String, TypeBuilder> typeBuilders = ObjectObjectOpenHashMap.newInstance();
parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes);
}
boolean success = false;
ParentChildAtomicFieldData data = null;
ParentChildFilteredTermsEnum termsEnum = new ParentChildFilteredTermsEnum(
new ParentChildIntersectTermsEnum(reader, UidFieldMapper.NAME, ParentFieldMapper.NAME),
parentTypes
);
ParentChildEstimator estimator = new ParentChildEstimator(breakerService.getBreaker(), termsEnum);
TermsEnum estimatedTermsEnum = estimator.beforeLoad(null);
ObjectObjectOpenHashMap<String, TypeBuilder> typeBuilders = ObjectObjectOpenHashMap.newInstance();
try {
try {
try {
DocsEnum docsEnum = null;
for (BytesRef term = estimatedTermsEnum.next(); term != null; term = estimatedTermsEnum.next()) {
// Usually this would be estimatedTermsEnum, but the
// abstract TermsEnum class does not support the .type()
// and .id() methods, so we skip using the wrapped
// TermsEnum and delegate directly to the
// ParentChildFilteredTermsEnum that was originally wrapped
String type = termsEnum.type();
TypeBuilder typeBuilder = typeBuilders.get(type);
if (typeBuilder == null) {
typeBuilders.put(type, typeBuilder = new TypeBuilder(acceptableTransientOverheadRatio, reader));
}
BytesRef id = termsEnum.id();
final long termOrd = typeBuilder.builder.nextOrdinal();
assert termOrd == typeBuilder.termOrdToBytesOffset.size();
typeBuilder.termOrdToBytesOffset.add(typeBuilder.bytes.copyUsingLengthPrefix(id));
docsEnum = estimatedTermsEnum.docs(null, docsEnum, DocsEnum.FLAG_NONE);
for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
typeBuilder.builder.addDoc(docId);
}
DocsEnum docsEnum = null;
for (BytesRef term = estimatedTermsEnum.next(); term != null; term = estimatedTermsEnum.next()) {
// Usually this would be estimatedTermsEnum, but the
// abstract TermsEnum class does not support the .type()
// and .id() methods, so we skip using the wrapped
// TermsEnum and delegate directly to the
// ParentChildFilteredTermsEnum that was originally wrapped
String type = termsEnum.type();
TypeBuilder typeBuilder = typeBuilders.get(type);
if (typeBuilder == null) {
typeBuilders.put(type, typeBuilder = new TypeBuilder(acceptableTransientOverheadRatio, reader));
}
ImmutableOpenMap.Builder<String, PagedBytesAtomicFieldData> typeToAtomicFieldData = ImmutableOpenMap.builder(typeBuilders.size());
for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) {
final long sizePointer = cursor.value.bytes.getPointer();
PagedBytes.Reader bytesReader = cursor.value.bytes.freeze(true);
final Ordinals ordinals = cursor.value.builder.build(fieldDataType.getSettings());
typeToAtomicFieldData.put(
cursor.key,
new PagedBytesAtomicFieldData(bytesReader, sizePointer, cursor.value.termOrdToBytesOffset, ordinals)
);
}
data = new ParentChildAtomicFieldData(typeToAtomicFieldData.build());
} finally {
for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) {
cursor.value.builder.close();
BytesRef id = termsEnum.id();
final long termOrd = typeBuilder.builder.nextOrdinal();
assert termOrd == typeBuilder.termOrdToBytesOffset.size();
typeBuilder.termOrdToBytesOffset.add(typeBuilder.bytes.copyUsingLengthPrefix(id));
docsEnum = estimatedTermsEnum.docs(null, docsEnum, DocsEnum.FLAG_NONE);
for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
typeBuilder.builder.addDoc(docId);
}
}
success = true;
return data;
ImmutableOpenMap.Builder<String, AtomicOrdinalsFieldData> typeToAtomicFieldData = ImmutableOpenMap.builder(typeBuilders.size());
for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) {
PagedBytes.Reader bytesReader = cursor.value.bytes.freeze(true);
final Ordinals ordinals = cursor.value.builder.build(fieldDataType.getSettings());
typeToAtomicFieldData.put(
cursor.key,
new PagedBytesAtomicFieldData(bytesReader, cursor.value.termOrdToBytesOffset, ordinals)
);
}
data = new ParentChildAtomicFieldData(typeToAtomicFieldData.build());
} finally {
if (success) {
estimator.afterLoad(estimatedTermsEnum, data.ramBytesUsed());
} else {
estimator.afterLoad(estimatedTermsEnum, 0);
for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) {
cursor.value.builder.close();
}
}
success = true;
return data;
} finally {
if (success) {
estimator.afterLoad(estimatedTermsEnum, data.ramBytesUsed());
} else {
estimator.afterLoad(estimatedTermsEnum, 0);
}
}
}
public WithOrdinals getGlobalParentChild(String type, IndexReader indexReader) {
ParentTypesGlobalOrdinalsLoading loading = new ParentTypesGlobalOrdinalsLoading();
ParentChildGlobalOrdinalsIndexFieldData holder = (ParentChildGlobalOrdinalsIndexFieldData) loading.loadGlobal(indexReader);
return holder.type(type);
}
@Override
public void beforeCreate(DocumentMapper mapper) {
synchronized (lock) {
@ -208,9 +197,9 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
@Override
public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService,
MapperService mapperService, GlobalOrdinalsBuilder globalOrdinalBuilder) {
MapperService mapperService) {
return new ParentChildIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache,
mapperService, breakerService, globalOrdinalBuilder);
mapperService, breakerService);
}
}
@ -263,59 +252,142 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
}
}
private class ParentTypesGlobalOrdinalsLoading implements WithOrdinals {
public ParentTypesGlobalOrdinalsLoading() {
@Override
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
}
@Override
public AtomicFieldData.WithOrdinals load(AtomicReaderContext context) {
throw new ElasticsearchIllegalStateException("Shouldn't be invoked");
}
@Override
public AtomicFieldData.WithOrdinals loadDirect(AtomicReaderContext context) {
throw new ElasticsearchIllegalStateException("Shouldn't be invoked");
}
@Override
public WithOrdinals loadGlobal(IndexReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
ImmutableOpenMap.Builder<String, WithOrdinals> globalIfdPerType = ImmutableOpenMap.builder();
for (BytesRef parentType : parentTypes) {
PerType perType = new PerType(parentType.utf8ToString());
globalIfdPerType.put(perType.type, perType);
}
return new ParentChildGlobalOrdinalsIndexFieldData(globalIfdPerType.build(), 0);
try {
return cache.load(indexReader, this);
} catch (Throwable e) {
if (e instanceof ElasticsearchException) {
throw (ElasticsearchException) e;
} else {
throw new ElasticsearchException(e.getMessage(), e);
}
}
}
try {
return cache.load(indexReader, this);
} catch (Throwable e) {
if (e instanceof ElasticsearchException) {
throw (ElasticsearchException) e;
} else {
throw new ElasticsearchException(e.getMessage(), e);
@Override
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
final long startTime = System.nanoTime();
final Map<String, SortedDocValues[]> types = new HashMap<>();
synchronized (lock) {
for (BytesRef type : parentTypes) {
final SortedDocValues[] values = new SortedDocValues[indexReader.leaves().size()];
Arrays.fill(values, DocValues.emptySorted());
types.put(type.utf8ToString(), values);
}
}
for (Map.Entry<String, SortedDocValues[]> entry : types.entrySet()) {
final String parentType = entry.getKey();
final SortedDocValues[] values = entry.getValue();
for (AtomicReaderContext context : indexReader.leaves()) {
SortedDocValues vals = load(context).getOrdinalsValues(parentType);
if (vals != null) {
values[context.ord] = vals;
}
}
}
@Override
public WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception {
ImmutableOpenMap.Builder<String, WithOrdinals> globalIfdPerType = ImmutableOpenMap.builder();
long memorySizeInBytes = 0;
for (BytesRef parentType : parentTypes) {
PerType perType = new PerType(parentType.utf8ToString());
GlobalOrdinalsIndexFieldData globalIfd = (GlobalOrdinalsIndexFieldData) globalOrdinalsBuilder.build(indexReader, perType, indexSettings, breakerService);
globalIfdPerType.put(perType.type, globalIfd);
memorySizeInBytes += globalIfd.ramBytesUsed();
long ramBytesUsed = 0;
@SuppressWarnings("unchecked")
final Map<String, SortedDocValues>[] global = new Map[indexReader.leaves().size()];
for (Map.Entry<String, SortedDocValues[]> entry : types.entrySet()) {
final String parentType = entry.getKey();
final SortedDocValues[] values = entry.getValue();
final XOrdinalMap ordinalMap = XOrdinalMap.build(null, entry.getValue(), PackedInts.DEFAULT);
ramBytesUsed += ordinalMap.ramBytesUsed();
for (int i = 0; i < values.length; ++i) {
final SortedDocValues segmentValues = values[i];
final LongValues globalOrds = ordinalMap.getGlobalOrds(i);
final SortedDocValues globalSortedValues = new SortedDocValues() {
@Override
public BytesRef lookupOrd(int ord) {
final int segmentNum = ordinalMap.getFirstSegmentNumber(ord);
final int segmentOrd = (int) ordinalMap.getFirstSegmentOrd(ord);
return values[segmentNum].lookupOrd(segmentOrd);
}
@Override
public int getValueCount() {
return (int) ordinalMap.getValueCount();
}
@Override
public int getOrd(int docID) {
final int segmentOrd = segmentValues.getOrd(docID);
// TODO: is there a way we can get rid of this branch?
if (segmentOrd >= 0) {
return (int) globalOrds.get(segmentOrd);
} else {
return segmentOrd;
}
}
};
Map<String, SortedDocValues> perSegmentGlobal = global[i];
if (perSegmentGlobal == null) {
perSegmentGlobal = new HashMap<>(1);
global[i] = perSegmentGlobal;
}
perSegmentGlobal.put(parentType, globalSortedValues);
}
}
breakerService.getBreaker().addWithoutBreaking(ramBytesUsed);
if (logger.isDebugEnabled()) {
logger.debug(
"Global-ordinals[_parent] took {}",
new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
);
}
return new GlobalFieldData(indexReader, global, ramBytesUsed);
}
private class GlobalFieldData implements IndexParentChildFieldData, Accountable {
private final AtomicParentChildFieldData[] atomicFDs;
private final IndexReader reader;
private final long ramBytesUsed;
GlobalFieldData(IndexReader reader, final Map<String, SortedDocValues>[] globalValues, long ramBytesUsed) {
this.reader = reader;
this.ramBytesUsed = ramBytesUsed;
this.atomicFDs = new AtomicParentChildFieldData[globalValues.length];
for (int i = 0; i < globalValues.length; ++i) {
final int ord = i;
atomicFDs[i] = new AbstractAtomicParentChildFieldData() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public void close() {
}
@Override
public Set<String> types() {
return Collections.unmodifiableSet(globalValues[ord].keySet());
}
@Override
public SortedDocValues getOrdinalsValues(String type) {
SortedDocValues dv = globalValues[ord].get(type);
if (dv == null) {
dv = DocValues.emptySorted();
}
return dv;
}
};
}
return new ParentChildGlobalOrdinalsIndexFieldData(globalIfdPerType.build(), memorySizeInBytes);
}
@Override
public FieldMapper.Names getFieldNames() {
public Names getFieldNames() {
return ParentChildIndexFieldData.this.getFieldNames();
}
@ -325,21 +397,30 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
}
@Override
public boolean valuesOrdered() {
return ParentChildIndexFieldData.this.valuesOrdered();
public AtomicParentChildFieldData load(AtomicReaderContext context) {
assert context.reader().getCoreCacheKey() == reader.leaves().get(context.ord).reader().getCoreCacheKey();
return atomicFDs[context.ord];
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
throw new UnsupportedOperationException("Sort not supported on PerParentTypeGlobalOrdinals...");
public AtomicParentChildFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue,
MultiValueMode sortMode) {
throw new UnsupportedOperationException("No sorting on global ords");
}
@Override
public void clear() {
ParentChildIndexFieldData.this.clear();
}
@Override
public void clear(IndexReader reader) {
ParentChildIndexFieldData.this.clear(reader);
}
@Override
@ -347,60 +428,24 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
return ParentChildIndexFieldData.this.index();
}
private final class PerType extends ParentTypesGlobalOrdinalsLoading {
private final String type;
public PerType(String type) {
this.type = type;
}
@Override
public AtomicFieldData.WithOrdinals load(AtomicReaderContext context) {
return loadDirect(context);
}
@Override
public AtomicFieldData.WithOrdinals loadDirect(AtomicReaderContext context) {
ParentChildAtomicFieldData parentChildAtomicFieldData = ParentChildIndexFieldData.this.load(context);
AtomicFieldData.WithOrdinals typeAfd = parentChildAtomicFieldData.getAtomicFieldData(type);
if(typeAfd != null) {
return typeAfd;
} else {
return AtomicFieldData.WithOrdinals.EMPTY;
}
}
@Override
public WithOrdinals loadGlobal(IndexReader indexReader) {
return this;
}
@Override
public WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception {
return this;
}
}
}
// Effectively this is a cache key for in the field data cache
private final class ParentChildGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData {
private final ImmutableOpenMap<String, WithOrdinals> typeGlobalOrdinals;
private ParentChildGlobalOrdinalsIndexFieldData(ImmutableOpenMap<String, WithOrdinals> typeGlobalOrdinals, long memorySizeInBytes) {
super(ParentChildIndexFieldData.this.index(), ParentChildIndexFieldData.this.indexSettings, ParentChildIndexFieldData.this.getFieldNames(), ParentChildIndexFieldData.this.getFieldDataType(), memorySizeInBytes);
this.typeGlobalOrdinals = typeGlobalOrdinals;
@Override
public long ramBytesUsed() {
return ramBytesUsed;
}
@Override
public AtomicFieldData.WithOrdinals load(AtomicReaderContext context) {
throw new ElasticsearchIllegalStateException("Can't use directly");
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
if (indexReader.getCoreCacheKey() == reader.getCoreCacheKey()) {
return this;
}
throw new ElasticsearchIllegalStateException();
}
public WithOrdinals type(String type) {
return typeGlobalOrdinals.get(type);
@Override
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
return loadGlobal(indexReader);
}
}
}

176
src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVAtomicFieldData.java
View File

@ -1,176 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import java.io.IOException;
/**
* {@link AtomicFieldData} impl based on Lucene's {@link SortedSetDocValues}.
* <p><b>Implementation note</b>: Lucene's ordinal for unset values is -1 whereas Elasticsearch's is 0, this is why there are all
* these +1 to translate from Lucene's ordinals to ES's.
*/
abstract class SortedSetDVAtomicFieldData {
private final AtomicReader reader;
private final String field;
private final boolean multiValued;
private final long valueCount;
SortedSetDVAtomicFieldData(AtomicReader reader, String field) {
this.reader = reader;
this.field = field;
SortedSetDocValues dv = getValuesNoException(reader, field);
this.multiValued = DocValues.unwrapSingleton(dv) == null;
this.valueCount = dv.getValueCount();
}
public boolean isMultiValued() {
return multiValued;
}
public long getNumberUniqueValues() {
return valueCount;
}
public long ramBytesUsed() {
// There is no API to access memory usage per-field and RamUsageEstimator can't help since there are often references
// from a per-field instance to all other instances handled by the same format
return -1L;
}
public void close() {
// no-op
}
public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getBytesValues() {
final SortedSetDocValues values = getValuesNoException(reader, field);
if (values instanceof RandomAccessOrds) {
return new RandomAccessSortedSetValues((RandomAccessOrds)values, multiValued);
} else {
return new SortedSetValues(values, multiValued);
}
}
public TermsEnum getTermsEnum() {
return getValuesNoException(reader, field).termsEnum();
}
private static SortedSetDocValues getValuesNoException(AtomicReader reader, String field) {
try {
return DocValues.getSortedSet(reader, field);
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("Couldn't load doc values", e);
}
}
private final static class RandomAccessSortedSetValues extends BytesValues.WithOrdinals {
private final RandomAccessOrds values;
private int index = 0;
RandomAccessSortedSetValues(RandomAccessOrds values, boolean multiValued) {
super(multiValued);
this.values = values;
}
@Override
public long getMaxOrd() {
return values.getValueCount();
}
@Override
public long getOrd(int docId) {
values.setDocument(docId);
return values.nextOrd();
}
@Override
public long nextOrd() {
return values.ordAt(index++);
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.lookupOrd(ord);
}
@Override
public int setDocument(int docId) {
values.setDocument(docId);
index = 0;
return values.cardinality();
}
}
private final static class SortedSetValues extends BytesValues.WithOrdinals {
private final SortedSetDocValues values;
private long[] ords;
private int ordIndex = Integer.MAX_VALUE;
SortedSetValues(SortedSetDocValues values, boolean multiValued) {
super(multiValued);
this.values = values;
ords = new long[0];
}
@Override
public long getMaxOrd() {
return values.getValueCount();
}
@Override
public long getOrd(int docId) {
values.setDocument(docId);
return values.nextOrd();
}
@Override
public long nextOrd() {
assert ordIndex < ords.length;
return ords[ordIndex++];
}
@Override
public int setDocument(int docId) {
// For now, we consume all ords and pass them to the iter instead of doing it in a streaming way because Lucene's
// SORTED_SET doc values are cached per thread, you can't have a fully independent instance
values.setDocument(docId);
int i = 0;
for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
ords = ArrayUtil.grow(ords, i + 1);
ords[i++] = ord;
}
ordIndex = 0;
return i;
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.lookupOrd(ord);
}
}
}

35
src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
View File

@ -20,26 +20,43 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
import org.elasticsearch.index.fielddata.FieldData;
import java.io.IOException;
/**
* An {@link AtomicFieldData} implementation that uses Lucene {@link org.apache.lucene.index.SortedSetDocValues}.
*/
public final class SortedSetDVBytesAtomicFieldData extends SortedSetDVAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinalsFieldData {
/* NOTE: This class inherits the methods getBytesValues() and getHashedBytesValues()
* from SortedSetDVAtomicFieldData. This can cause confusion since the are
* part of the interface this class implements.*/
private final AtomicReader reader;
private final String field;
SortedSetDVBytesAtomicFieldData(AtomicReader reader, String field) {
super(reader, field);
this.reader = reader;
this.field = field;
}
@Override
public Strings getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
public RandomAccessOrds getOrdinalsValues() {
try {
return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field));
} catch (IOException e) {
throw new ElasticsearchIllegalStateException("cannot load docvalues", e);
}
}
@Override
public void close() {
}
@Override
public long ramBytesUsed() {
return -1; // unknown
}
}

27
src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesIndexFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
View File

@ -24,51 +24,42 @@ import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
public class SortedSetDVBytesIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData.WithOrdinals<SortedSetDVBytesAtomicFieldData> {
public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData implements IndexOrdinalsFieldData {
private final Settings indexSettings;
private final IndexFieldDataCache cache;
private final GlobalOrdinalsBuilder globalOrdinalsBuilder;
private final CircuitBreakerService breakerService;
public SortedSetDVBytesIndexFieldData(Index index, IndexFieldDataCache cache, Settings indexSettings, Names fieldNames, GlobalOrdinalsBuilder globalOrdinalBuilder, CircuitBreakerService breakerService, FieldDataType fieldDataType) {
public SortedSetDVOrdinalsIndexFieldData(Index index, IndexFieldDataCache cache, Settings indexSettings, Names fieldNames, CircuitBreakerService breakerService, FieldDataType fieldDataType) {
super(index, fieldNames, fieldDataType);
this.indexSettings = indexSettings;
this.cache = cache;
this.globalOrdinalsBuilder = globalOrdinalBuilder;
this.breakerService = breakerService;
}
@Override
public boolean valuesOrdered() {
return true;
}
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode);
}
@Override
public SortedSetDVBytesAtomicFieldData load(AtomicReaderContext context) {
public AtomicOrdinalsFieldData load(AtomicReaderContext context) {
return new SortedSetDVBytesAtomicFieldData(context.reader(), fieldNames.indexName());
}
@Override
public SortedSetDVBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public AtomicOrdinalsFieldData loadDirect(AtomicReaderContext context) throws Exception {
return load(context);
}
@Override
public WithOrdinals loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
@ -85,7 +76,7 @@ public class SortedSetDVBytesIndexFieldData extends DocValuesIndexFieldData impl
}
@Override
public WithOrdinals localGlobalDirect(IndexReader indexReader) throws Exception {
return globalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService);
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
}
}

2
src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -395,7 +395,7 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
}
Filter filter = NumericRangeFieldDataFilter.newLongRange(
(IndexNumericFieldData<?>) parseContext.getForField(this), lowerVal,upperVal, includeLower, includeUpper
(IndexNumericFieldData) parseContext.getForField(this), lowerVal,upperVal, includeLower, includeUpper
);
if (!cache) {
// We don't cache range filter if `now` date expression is used and also when a compound filter wraps

12
src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
View File

@ -26,9 +26,9 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -44,10 +44,10 @@ final class QueriesLoaderCollector extends Collector {
private final Map<BytesRef, Query> queries = Maps.newHashMap();
private final JustSourceFieldsVisitor fieldsVisitor = new JustSourceFieldsVisitor();
private final PercolatorQueriesRegistry percolator;
private final IndexFieldData idFieldData;
private final IndexFieldData<?> idFieldData;
private final ESLogger logger;
private BytesValues idValues;
private SortedBinaryDocValues idValues;
private AtomicReader reader;
QueriesLoaderCollector(PercolatorQueriesRegistry percolator, ESLogger logger, MapperService mapperService, IndexFieldDataService indexFieldDataService) {
@ -65,8 +65,10 @@ final class QueriesLoaderCollector extends Collector {
public void collect(int doc) throws IOException {
// the _source is the query
if (idValues.setDocument(doc) > 0) {
BytesRef id = idValues.nextValue();
idValues.setDocument(doc);
if (idValues.count() > 0) {
assert idValues.count() == 1;
BytesRef id = idValues.valueAt(0);
fieldsVisitor.reset();
reader.document(doc, fieldsVisitor);

Some files were not shown because too many files have changed in this diff Show More