Fielddata: Merge ordinals APIs into BytesValues.WithOrdinals.

Mid-term we should switch from `BytesValues` to Lucene's doc values APIs, in
particular the `SortedSetDocValues` class. While `BytesValues.WithOrdinals` and
SortedSetDocValues expose the same functionality, `BytesValues.WithOrdinals`
exposes its ordinals via a different `Ordinals.Docs` object while
`SortedSetDocValues` exposes them on the same object as the one that holds the
values. This commit merges ordinals into `BytesValues.WithOrdinals` in order to
make both classes even closer.

Global ordinals were a bit tricky to migrate so I just changed them to use
Lucene's OrdinalMap that will soon (LUCENE-5767, scheduled for 4.9) have the
same optimizations as our global ordinals.

Close #6524
This commit is contained in:
Adrien Grand 2014-06-14 00:39:40 +02:00
parent 703dbff83d
commit 9b02b5061b
66 changed files with 627 additions and 1751 deletions

View File

@ -45,10 +45,10 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
if (isFloat) {
final DoubleValues values = getDoubleValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
this.docId = docId;
return values.setDocument(docId);
}
@ -67,10 +67,10 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
} else {
final LongValues values = getLongValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
this.docId = docId;
return values.setDocument(docId);
}

View File

@ -20,29 +20,20 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
import org.elasticsearch.index.fielddata.plain.AtomicFieldDataWithOrdinalsTermsEnum;
/**
* The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
*/
public interface AtomicFieldData<Script extends ScriptDocValues> extends RamUsage {
/**
* If this method returns false, this means that no document has multiple values. However this method may return true even if all
* documents are single-valued. So this method is useful for performing optimizations when the single-value case makes the problem
* simpler but cannot be used to actually check whether this instance is multi-valued.
*/
boolean isMultiValued();
/**
* An upper limit of the number of unique values in this atomic field data.
*/
long getNumberUniqueValues();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
BytesValues getBytesValues();
/**
* Returns a "scripting" based values.
*/
@ -55,6 +46,65 @@ public interface AtomicFieldData<Script extends ScriptDocValues> extends RamUsag
interface WithOrdinals<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
public static final WithOrdinals<ScriptDocValues.Strings> EMPTY = new WithOrdinals<ScriptDocValues.Strings>() {
@Override
public Strings getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public void close() {
}
@Override
public long getMemorySizeInBytes() {
return 0;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new BytesValues.WithOrdinals(false) {
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public long nextOrd() {
return MISSING_ORDINAL;
}
@Override
public BytesRef getValueByOrd(long ord) {
throw new UnsupportedOperationException();
}
@Override
public long getOrd(int docId) {
return MISSING_ORDINAL;
}
@Override
public long getMaxOrd() {
return 0;
}
@Override
public long currentOrd() {
return MISSING_ORDINAL;
}
};
}
@Override
public TermsEnum getTermsEnum() {
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
}
};
/**
* Use a non thread safe (lightweight) view of the values as bytes.
* @param needsHashes

View File

@ -32,10 +32,10 @@ public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> im
public BytesValues getBytesValues() {
final GeoPointValues values = getGeoPointValues();
return new BytesValues(values.isMultiValued()) {
private final BytesRef scratch = new BytesRef();
@Override
public int setDocument(int docId) {
this.docId = docId;
return values.setDocument(docId);
}

View File

@ -19,10 +19,9 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
/**
* A state-full lightweight per document set of <code>byte[]</code> values.
@ -44,11 +43,7 @@ public abstract class BytesValues {
*/
public static final BytesValues EMPTY = new Empty();
private boolean multiValued;
protected final BytesRef scratch = new BytesRef();
protected int docId = -1;
private final boolean multiValued;
/**
* Creates a new {@link BytesValues} instance
@ -114,20 +109,41 @@ public abstract class BytesValues {
*/
public static abstract class WithOrdinals extends BytesValues {
protected final Docs ordinals;
public static final long MIN_ORDINAL = 0;
public static final long MISSING_ORDINAL = SortedSetDocValues.NO_MORE_ORDS;
protected WithOrdinals(Ordinals.Docs ordinals) {
super(ordinals.isMultiValued());
this.ordinals = ordinals;
protected WithOrdinals(boolean multiValued) {
super(multiValued);
}
/**
* Returns the associated ordinals instance.
* @return the associated ordinals instance.
* Returns total unique ord count;
*/
public Ordinals.Docs ordinals() {
return ordinals;
}
public abstract long getMaxOrd();
/**
* The ordinal that maps to the relevant docId. If it has no value, returns
* <tt>0</tt>.
*/
public abstract long getOrd(int docId);
/**
* Returns the next ordinal for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
*
* Note: This method will never return <tt>0</tt>.
*
* @return the next ordinal for the current docID set to {@link #setDocument(int)}.
*/
public abstract long nextOrd();
/**
* Returns the current ordinal in the iteration
* @return the current ordinal in the iteration
*/
public abstract long currentOrd();
/**
* Returns the value for the given ordinal.
@ -137,18 +153,9 @@ public abstract class BytesValues {
*/
public abstract BytesRef getValueByOrd(long ord);
@Override
public int setDocument(int docId) {
this.docId = docId;
int length = ordinals.setDocument(docId);
assert (ordinals.getOrd(docId) != Ordinals.MISSING_ORDINAL) == length > 0 : "Doc: [" + docId + "] hasValue: [" + (ordinals.getOrd(docId) != Ordinals.MISSING_ORDINAL) + "] but length is [" + length + "]";
return length;
}
@Override
public BytesRef nextValue() {
assert docId != -1;
return getValueByOrd(ordinals.nextOrd());
return getValueByOrd(nextOrd());
}
}

View File

@ -20,8 +20,6 @@
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
/**
* A state-full lightweight per document set of <code>double</code> values.
@ -100,21 +98,13 @@ public abstract class DoubleValues {
*/
public static abstract class WithOrdinals extends DoubleValues {
protected final Docs ordinals;
protected final BytesValues.WithOrdinals ordinals;
protected WithOrdinals(Ordinals.Docs ordinals) {
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.ordinals = ordinals;
}
/**
* Returns the associated ordinals instance.
* @return the associated ordinals instance.
*/
public Docs ordinals() {
return ordinals;
}
/**
* Returns the value for the given ordinal.
* @param ord the ordinal to lookup.

View File

@ -20,8 +20,6 @@
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
/**
* A state-full lightweight per document set of <code>long</code> values.
@ -101,21 +99,13 @@ public abstract class LongValues {
*/
public static abstract class WithOrdinals extends LongValues {
protected final Docs ordinals;
protected final BytesValues.WithOrdinals ordinals;
protected WithOrdinals(Ordinals.Docs ordinals) {
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.ordinals = ordinals;
}
/**
* Returns the associated ordinals instance.
* @return the associated ordinals instance.
*/
public Docs ordinals() {
return this.ordinals;
}
/**
* Returns the value for the given ordinal.
* @param ord the ordinal to lookup.

View File

@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -142,13 +141,11 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
}
class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
final Ordinals.Docs readerOrds;
final BytesValues.WithOrdinals termsIndex;
public PerSegmentComparator(BytesValues.WithOrdinals termsIndex) {
this.readerOrds = termsIndex.ordinals();
this.termsIndex = termsIndex;
if (readerOrds.getMaxOrd() > Long.MAX_VALUE / 4) {
if (termsIndex.getMaxOrd() > Long.MAX_VALUE / 4) {
throw new IllegalStateException("Current terms index pretends it has more than " + (Long.MAX_VALUE / 4) + " ordinals, which is unsupported by this impl");
}
}
@ -192,21 +189,21 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
}
protected long getOrd(int doc) {
return readerOrds.getOrd(doc);
return termsIndex.getOrd(doc);
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final long docOrd = getOrd(doc);
final long comparableOrd = docOrd == Ordinals.MISSING_ORDINAL ? missingOrd : docOrd << 2;
final long comparableOrd = docOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? missingOrd : docOrd << 2;
return Long.compare(bottomOrd, comparableOrd);
}
@Override
public int compareTop(int doc) throws IOException {
final long ord = getOrd(doc);
if (ord == Ordinals.MISSING_ORDINAL) {
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
return compareTopMissing();
} else {
final long comparableOrd = ord << 2;
@ -233,7 +230,7 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
@Override
public void copy(int slot, int doc) {
final long ord = getOrd(doc);
if (ord == Ordinals.MISSING_ORDINAL) {
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
ords[slot] = missingOrd;
values[slot] = missingValue;
} else {
@ -259,13 +256,13 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
private boolean consistentInsertedOrd(BytesValues.WithOrdinals termsIndex, long ord, BytesRef value) {
final long previousOrd = ord >> 2;
final long nextOrd = previousOrd + 1;
final BytesRef previous = previousOrd == Ordinals.MISSING_ORDINAL ? null : termsIndex.getValueByOrd(previousOrd);
final BytesRef previous = previousOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? null : termsIndex.getValueByOrd(previousOrd);
if ((ord & 3) == 0) { // there was an existing ord with the inserted value
assert compareValues(previous, value) == 0;
} else {
assert compareValues(previous, value) < 0;
}
if (nextOrd < termsIndex.ordinals().getMaxOrd()) {
if (nextOrd < termsIndex.getMaxOrd()) {
final BytesRef next = termsIndex.getValueByOrd(nextOrd);
assert compareValues(value, next) < 0;
}
@ -276,10 +273,10 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
private long ordInCurrentReader(BytesValues.WithOrdinals termsIndex, BytesRef value) {
final long ord;
if (value == null) {
ord = Ordinals.MISSING_ORDINAL << 2;
ord = BytesValues.WithOrdinals.MISSING_ORDINAL << 2;
} else {
final long docOrd = binarySearch(termsIndex, value);
if (docOrd >= Ordinals.MIN_ORDINAL) {
if (docOrd >= BytesValues.WithOrdinals.MIN_ORDINAL) {
// value exists in the current segment
ord = docOrd << 2;
} else {
@ -295,16 +292,14 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
termsIndex = indexFieldData.load(context).getBytesValues();
assert termsIndex.ordinals() != null;
missingOrd = ordInCurrentReader(termsIndex, missingValue);
assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
FieldComparator<BytesRef> perSegComp = null;
assert termsIndex.ordinals() != null;
if (termsIndex.isMultiValued()) {
perSegComp = new PerSegmentComparator(termsIndex) {
@Override
protected long getOrd(int doc) {
return getRelevantOrd(readerOrds, doc, sortMode);
return getRelevantOrd(termsIndex, doc, sortMode);
}
};
} else {
@ -359,12 +354,12 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
}
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
return binarySearch(a, key, Ordinals.MIN_ORDINAL, a.ordinals().getMaxOrd() - 1);
return binarySearch(a, key, BytesValues.WithOrdinals.MIN_ORDINAL, a.getMaxOrd() - 1);
}
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key, long low, long high) {
assert low != Ordinals.MISSING_ORDINAL;
assert high == Ordinals.MISSING_ORDINAL || (a.getValueByOrd(high) == null | a.getValueByOrd(high) != null); // make sure we actually can get these values
assert low != BytesValues.WithOrdinals.MISSING_ORDINAL;
assert high == BytesValues.WithOrdinals.MISSING_ORDINAL || (a.getValueByOrd(high) == null | a.getValueByOrd(high) != null); // make sure we actually can get these values
assert low == high + 1 || a.getValueByOrd(low) == null | a.getValueByOrd(low) != null;
while (low <= high) {
long mid = (low + high) >>> 1;
@ -386,15 +381,15 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
return -(low + 1);
}
static long getRelevantOrd(Ordinals.Docs readerOrds, int docId, MultiValueMode sortMode) {
static long getRelevantOrd(BytesValues.WithOrdinals readerOrds, int docId, MultiValueMode sortMode) {
int length = readerOrds.setDocument(docId);
long relevantVal = sortMode.startLong();
long result = Ordinals.MISSING_ORDINAL;
long result = BytesValues.WithOrdinals.MISSING_ORDINAL;
assert sortMode == MultiValueMode.MAX || sortMode == MultiValueMode.MIN;
for (int i = 0; i < length; i++) {
result = relevantVal = sortMode.apply(readerOrds.nextOrd(), relevantVal);
}
assert result >= Ordinals.MISSING_ORDINAL;
assert result >= BytesValues.WithOrdinals.MISSING_ORDINAL;
assert result < readerOrds.getMaxOrd();
return result;
// Enable this when the api can tell us that the ords per doc are ordered

View File

@ -1,95 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.RamUsageEstimator;
/**
* Ordinals that effectively are single valued and map "one to one" to the
* doc ids. Note, the docId is incremented by 1 to get the ordinal, since 0
* denotes an empty value.
*/
public class DocIdOrdinals implements Ordinals {
private final int numDocs;
/**
* Constructs a new doc id ordinals.
*/
public DocIdOrdinals(int numDocs) {
this.numDocs = numDocs;
}
@Override
public long getMemorySizeInBytes() {
return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getMaxOrd() {
return Ordinals.MIN_ORDINAL + numDocs;
}
@Override
public Ordinals.Docs ordinals() {
return new Docs(this);
}
public static class Docs extends Ordinals.AbstractDocs {
private final LongsRef longsScratch = new LongsRef(new long[1], 0, 1);
private int docId = -1;
private long currentOrdinal = -1;
public Docs(DocIdOrdinals parent) {
super(parent);
}
@Override
public long getOrd(int docId) {
return currentOrdinal = docId + 1;
}
@Override
public long nextOrd() {
assert docId >= 0;
currentOrdinal = docId + 1;
docId = -1;
return currentOrdinal;
}
@Override
public int setDocument(int docId) {
this.docId = docId;
return 1;
}
@Override
public long currentOrd() {
return currentOrdinal;
}
}
}

View File

@ -1,75 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.ElasticsearchIllegalStateException;
/**
*/
public enum EmptyOrdinals implements Ordinals {
INSTANCE;
@Override
public long getMemorySizeInBytes() {
return 0;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getMaxOrd() {
return 0;
}
@Override
public Docs ordinals() {
return new Docs(this);
}
public static class Docs extends Ordinals.AbstractDocs {
public Docs(EmptyOrdinals parent) {
super(parent);
}
@Override
public long getOrd(int docId) {
return Ordinals.MISSING_ORDINAL;
}
@Override
public long nextOrd() {
throw new ElasticsearchIllegalStateException("Empty ordinals has no nextOrd");
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public long currentOrd() {
return Ordinals.MISSING_ORDINAL;
}
}
}

View File

@ -0,0 +1,87 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
/**
* A {@link BytesValues.WithOrdinals} implementation that returns ordinals that are global.
*/
public class GlobalOrdinalMapping extends BytesValues.WithOrdinals {
private final BytesValues.WithOrdinals values;
private final OrdinalMap ordinalMap;
private final BytesValues.WithOrdinals[] bytesValues;
private final int segmentIndex;
GlobalOrdinalMapping(OrdinalMap ordinalMap, BytesValues.WithOrdinals[] bytesValues, int segmentIndex) {
super(bytesValues[segmentIndex].isMultiValued());
this.values = bytesValues[segmentIndex];
this.segmentIndex = segmentIndex;
this.bytesValues = bytesValues;
this.ordinalMap = ordinalMap;
}
int readerIndex;
@Override
public BytesRef copyShared() {
return bytesValues[readerIndex].copyShared();
}
@Override
public long getMaxOrd() {
return ordinalMap.getValueCount();
}
public long getGlobalOrd(long segmentOrd) {
return segmentOrd == MISSING_ORDINAL ? MISSING_ORDINAL : ordinalMap.getGlobalOrd(segmentIndex, segmentOrd);
}
@Override
public long getOrd(int docId) {
return getGlobalOrd(values.getOrd(docId));
}
@Override
public long nextOrd() {
return getGlobalOrd(values.nextOrd());
}
@Override
public long currentOrd() {
return getGlobalOrd(values.currentOrd());
}
@Override
public int setDocument(int docId) {
return values.setDocument(docId);
}
@Override
public BytesRef getValueByOrd(long globalOrd) {
final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);
readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd);
return bytesValues[readerIndex].getValueByOrd(segmentOrd);
}
}

View File

@ -19,37 +19,23 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/**
*/
public class InternalGlobalOrdinalsBuilder extends AbstractIndexComponent implements GlobalOrdinalsBuilder {
public final static int ORDINAL_MAPPING_THRESHOLD_DEFAULT = 2048;
public final static String ORDINAL_MAPPING_THRESHOLD_KEY = "global_ordinals_compress_threshold";
public final static String ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY = "index." + ORDINAL_MAPPING_THRESHOLD_KEY;
public InternalGlobalOrdinalsBuilder(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
}
@ -59,367 +45,27 @@ public class InternalGlobalOrdinalsBuilder extends AbstractIndexComponent implem
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
// It makes sense to make the overhead ratio configurable for the mapping from segment ords to global ords
// However, other mappings are never the bottleneck and only used to get the original value from an ord, so
// it makes sense to force COMPACT for them
final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT);
final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer(PackedInts.COMPACT);
FieldDataType fieldDataType = indexFieldData.getFieldDataType();
int defaultThreshold = settings.getAsInt(ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder(
indexReader.leaves().size(), acceptableOverheadRatio, threshold
);
long currentGlobalOrdinal = 0;
final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
ordinalMappingBuilder.onOrdinal(leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
}
currentGlobalOrdinal++;
final AtomicFieldData.WithOrdinals<?>[] atomicFD = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
final TermsEnum[] subs = new TermsEnum[indexReader.leaves().size()];
for (int i = 0; i < indexReader.leaves().size(); ++i) {
atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i));
subs[i] = atomicFD[i].getTermsEnum();
}
// ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
long memorySizeInBytesCounter = 0;
globalOrdToFirstSegment.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
globalOrdToFirstSegmentDelta.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();
final long maxOrd = currentGlobalOrdinal;
OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
// add ram used for the main segmentOrd to globalOrd lookups
memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();
final long memorySizeInBytes = memorySizeInBytesCounter;
final OrdinalMap ordinalMap = new OrdinalMap(null, subs);
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
// this does include the [] from the array in the impl name
String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
logger.debug(
"Global-ordinals[{}][{}][{}] took {} ms",
implName,
"Global-ordinals[{}][{}] took {} ms",
indexFieldData.getFieldNames().fullName(),
maxOrd,
ordinalMap.getValueCount(),
(System.currentTimeMillis() - startTime)
);
}
return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(),
fieldDataType, withOrdinals, globalOrdToFirstSegment, globalOrdToFirstSegmentDelta,
segmentOrdToGlobalOrdLookups, memorySizeInBytes
indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes
);
}
public interface OrdinalMappingSource {
Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals);
}
public static abstract class GlobalOrdinalMapping implements Ordinals.Docs {
protected final Ordinals.Docs segmentOrdinals;
private final long memorySizeInBytes;
protected final long maxOrd;
protected long currentGlobalOrd;
private GlobalOrdinalMapping(Ordinals.Docs segmentOrdinals, long memorySizeInBytes, long maxOrd) {
this.segmentOrdinals = segmentOrdinals;
this.memorySizeInBytes = memorySizeInBytes;
this.maxOrd = maxOrd;
}
@Override
public final long getMaxOrd() {
return maxOrd;
}
@Override
public final boolean isMultiValued() {
return segmentOrdinals.isMultiValued();
}
@Override
public final int setDocument(int docId) {
return segmentOrdinals.setDocument(docId);
}
@Override
public final long currentOrd() {
return currentGlobalOrd;
}
@Override
public final long getOrd(int docId) {
long segmentOrd = segmentOrdinals.getOrd(docId);
if (segmentOrd == Ordinals.MISSING_ORDINAL) {
return currentGlobalOrd = Ordinals.MISSING_ORDINAL;
} else {
return currentGlobalOrd = getGlobalOrd(segmentOrd);
}
}
@Override
public final long nextOrd() {
long segmentOrd = segmentOrdinals.nextOrd();
return currentGlobalOrd = getGlobalOrd(segmentOrd);
}
public abstract long getGlobalOrd(long segmentOrd);
}
private final static class OrdinalMappingSourceBuilder {
final MonotonicAppendingLongBuffer[] segmentOrdToGlobalOrdDeltas;
final float acceptableOverheadRatio;
final int numSegments;
final int threshold;
long memorySizeInBytesCounter;
private OrdinalMappingSourceBuilder(int numSegments, float acceptableOverheadRatio, int threshold) {
segmentOrdToGlobalOrdDeltas = new MonotonicAppendingLongBuffer[numSegments];
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
segmentOrdToGlobalOrdDeltas[i] = new MonotonicAppendingLongBuffer(acceptableOverheadRatio);
}
this.numSegments = numSegments;
this.acceptableOverheadRatio = acceptableOverheadRatio;
this.threshold = threshold;
}
public void onOrdinal(int readerIndex, long segmentOrdinal, long globalOrdinal) {
long delta = globalOrdinal - segmentOrdinal;
segmentOrdToGlobalOrdDeltas[readerIndex].add(delta);
}
public OrdinalMappingSource[] build(long maxOrd) {
// If we find out that there are less then predefined number of ordinals, it is better to put the the
// segment ordinal to global ordinal mapping in a packed ints, since the amount values are small and
// will most likely fit in the CPU caches and MonotonicAppendingLongBuffer's compression will just be
// unnecessary.
if (maxOrd <= threshold) {
// Rebuilding from MonotonicAppendingLongBuffer to PackedInts.Mutable is fast
PackedInts.Mutable[] newSegmentOrdToGlobalOrdDeltas = new PackedInts.Mutable[numSegments];
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
newSegmentOrdToGlobalOrdDeltas[i] = PackedInts.getMutable((int) segmentOrdToGlobalOrdDeltas[i].size(), PackedInts.bitsRequired(maxOrd), acceptableOverheadRatio);
}
for (int readerIndex = 0; readerIndex < segmentOrdToGlobalOrdDeltas.length; readerIndex++) {
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdDelta = segmentOrdToGlobalOrdDeltas[readerIndex];
for (long ordIndex = 0; ordIndex < segmentOrdToGlobalOrdDelta.size(); ordIndex++) {
long ordDelta = segmentOrdToGlobalOrdDelta.get(ordIndex);
newSegmentOrdToGlobalOrdDeltas[readerIndex].set((int) ordIndex, ordDelta);
}
}
PackedIntOrdinalMappingSource[] sources = new PackedIntOrdinalMappingSource[numSegments];
for (int i = 0; i < newSegmentOrdToGlobalOrdDeltas.length; i++) {
PackedInts.Reader segmentOrdToGlobalOrdDelta = newSegmentOrdToGlobalOrdDeltas[i];
if (segmentOrdToGlobalOrdDelta.size() == maxOrd) {
// This means that a segment contains all the value and in that case segment ordinals
// can be used as global ordinals. This will save an extra lookup per hit.
sources[i] = null;
} else {
long ramUsed = segmentOrdToGlobalOrdDelta.ramBytesUsed();
sources[i] = new PackedIntOrdinalMappingSource(segmentOrdToGlobalOrdDelta, ramUsed, maxOrd);
memorySizeInBytesCounter += ramUsed;
}
}
return sources;
} else {
OrdinalMappingSource[] sources = new OrdinalMappingSource[segmentOrdToGlobalOrdDeltas.length];
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdDeltas[i];
if (segmentOrdToGlobalOrdLookup.size() == maxOrd) {
// idem as above
sources[i] = null;
} else {
segmentOrdToGlobalOrdLookup.freeze();
long ramUsed = segmentOrdToGlobalOrdLookup.ramBytesUsed();
sources[i] = new CompressedOrdinalMappingSource(segmentOrdToGlobalOrdLookup, ramUsed, maxOrd);
memorySizeInBytesCounter += ramUsed;
}
}
return sources;
}
}
public long getMemorySizeInBytes() {
return memorySizeInBytesCounter;
}
}
private final static class CompressedOrdinalMappingSource implements OrdinalMappingSource {
private final MonotonicAppendingLongBuffer globalOrdinalMapping;
private final long memorySizeInBytes;
private final long maxOrd;
private CompressedOrdinalMappingSource(MonotonicAppendingLongBuffer globalOrdinalMapping, long memorySizeInBytes, long maxOrd) {
this.globalOrdinalMapping = globalOrdinalMapping;
this.memorySizeInBytes = memorySizeInBytes;
this.maxOrd = maxOrd;
}
@Override
public Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals) {
return new GlobalOrdinalsDocs(segmentOrdinals, globalOrdinalMapping, memorySizeInBytes, maxOrd);
}
private final static class GlobalOrdinalsDocs extends GlobalOrdinalMapping {
private final MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup;
private GlobalOrdinalsDocs(Ordinals.Docs segmentOrdinals, MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup, long memorySizeInBytes, long maxOrd) {
super(segmentOrdinals, memorySizeInBytes, maxOrd);
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
}
@Override
public long getGlobalOrd(long segmentOrd) {
return segmentOrd + segmentOrdToGlobalOrdLookup.get(segmentOrd);
}
}
}
private static final class PackedIntOrdinalMappingSource implements OrdinalMappingSource {
private final PackedInts.Reader segmentOrdToGlobalOrdLookup;
private final long memorySizeInBytes;
private final long maxOrd;
private PackedIntOrdinalMappingSource(PackedInts.Reader segmentOrdToGlobalOrdLookup, long memorySizeInBytes, long maxOrd) {
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
this.memorySizeInBytes = memorySizeInBytes;
this.maxOrd = maxOrd;
}
@Override
public Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals) {
return new GlobalOrdinalsDocs(segmentOrdinals, memorySizeInBytes, maxOrd, segmentOrdToGlobalOrdLookup);
}
private final static class GlobalOrdinalsDocs extends GlobalOrdinalMapping {
private final PackedInts.Reader segmentOrdToGlobalOrdLookup;
private GlobalOrdinalsDocs(Ordinals.Docs segmentOrdinals, long memorySizeInBytes, long maxOrd, PackedInts.Reader segmentOrdToGlobalOrdLookup) {
super(segmentOrdinals, memorySizeInBytes, maxOrd);
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
}
@Override
public long getGlobalOrd(long segmentOrd) {
return segmentOrd + segmentOrdToGlobalOrdLookup.get((int) segmentOrd);
}
}
}
private final static class TermIterator implements BytesRefIterator {
private final LeafSourceQueue sources;
private final List<LeafSource> competitiveLeafs = new ArrayList<>();
private TermIterator(IndexFieldData.WithOrdinals indexFieldData, List<AtomicReaderContext> leaves, AtomicFieldData.WithOrdinals[] withOrdinals) throws IOException {
this.sources = new LeafSourceQueue(leaves.size());
for (int i = 0; i < leaves.size(); i++) {
AtomicReaderContext atomicReaderContext = leaves.get(i);
AtomicFieldData.WithOrdinals afd = indexFieldData.load(atomicReaderContext);
withOrdinals[i] = afd;
LeafSource leafSource = new LeafSource(afd, atomicReaderContext);
if (leafSource.current != null) {
sources.add(leafSource);
}
}
}
public BytesRef next() throws IOException {
for (LeafSource top : competitiveLeafs) {
if (top.next() != null) {
sources.add(top);
}
}
competitiveLeafs.clear();
if (sources.size() == 0) {
return null;
}
do {
LeafSource competitiveLeaf = sources.pop();
competitiveLeafs.add(competitiveLeaf);
} while (sources.size() > 0 && competitiveLeafs.get(0).current.equals(sources.top().current));
return competitiveLeafs.get(0).current;
}
@Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
List<LeafSource> competitiveLeafs() throws IOException {
return competitiveLeafs;
}
int firstReaderIndex() {
return competitiveLeafs.get(0).context.ord;
}
long firstLocalOrdinal() throws IOException {
return competitiveLeafs.get(0).tenum.ord();
}
private static class LeafSource {
final TermsEnum tenum;
final AtomicReaderContext context;
BytesRef current;
private LeafSource(AtomicFieldData.WithOrdinals afd, AtomicReaderContext context) throws IOException {
this.tenum = afd.getTermsEnum();
this.context = context;
this.current = tenum.next();
}
BytesRef next() throws IOException {
return current = tenum.next();
}
}
private final static class LeafSourceQueue extends PriorityQueue<LeafSource> {
private final Comparator<BytesRef> termComp = BytesRef.getUTF8SortedAsUnicodeComparator();
LeafSourceQueue(int size) {
super(size);
}
@Override
protected boolean lessThan(LeafSource termsA, LeafSource termsB) {
final int cmp = termComp.compare(termsA.current, termsB.current);
if (cmp != 0) {
return cmp < 0;
} else {
return termsA.context.ord < termsB.context.ord;
}
}
}
}
}

View File

@ -19,16 +19,14 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.OrdinalMappingSource;
import org.elasticsearch.index.fielddata.plain.AtomicFieldDataWithOrdinalsTermsEnum;
import org.elasticsearch.index.mapper.FieldMapper;
@ -39,11 +37,11 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
private final Atomic[] atomicReaders;
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData.WithOrdinals[] segmentAfd, LongValues globalOrdToFirstSegment, LongValues globalOrdToFirstSegmentDelta, OrdinalMappingSource[] segmentOrdToGlobalOrds, long memorySizeInBytes) {
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData.WithOrdinals[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) {
super(index, settings, fieldNames, fieldDataType, memorySizeInBytes);
this.atomicReaders = new Atomic[segmentAfd.length];
for (int i = 0; i < segmentAfd.length; i++) {
atomicReaders[i] = new Atomic(segmentAfd[i], globalOrdToFirstSegment, globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrds[i]);
atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
}
}
@ -55,57 +53,27 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
private final class Atomic implements AtomicFieldData.WithOrdinals {
private final WithOrdinals afd;
private final OrdinalMappingSource segmentOrdToGlobalOrdLookup;
private final LongValues globalOrdToFirstSegment;
private final LongValues globalOrdToFirstSegmentDelta;
private final OrdinalMap ordinalMap;
private final int segmentIndex;
private Atomic(WithOrdinals afd, LongValues globalOrdToFirstSegment, LongValues globalOrdToFirstSegmentDelta, OrdinalMappingSource segmentOrdToGlobalOrdLookup) {
private Atomic(WithOrdinals afd, OrdinalMap ordinalMap, int segmentIndex) {
this.afd = afd;
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
this.globalOrdToFirstSegment = globalOrdToFirstSegment;
this.globalOrdToFirstSegmentDelta = globalOrdToFirstSegmentDelta;
this.ordinalMap = ordinalMap;
this.segmentIndex = segmentIndex;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
BytesValues.WithOrdinals values = afd.getBytesValues();
Ordinals.Docs segmentOrdinals = values.ordinals();
final Ordinals.Docs globalOrdinals;
if (segmentOrdToGlobalOrdLookup != null) {
globalOrdinals = segmentOrdToGlobalOrdLookup.globalOrdinals(segmentOrdinals);
} else {
globalOrdinals = segmentOrdinals;
final BytesValues.WithOrdinals values = afd.getBytesValues();
if (values.getMaxOrd() == ordinalMap.getValueCount()) {
// segment ordinals match global ordinals
return values;
}
final BytesValues.WithOrdinals[] bytesValues = new BytesValues.WithOrdinals[atomicReaders.length];
for (int i = 0; i < bytesValues.length; i++) {
bytesValues[i] = atomicReaders[i].afd.getBytesValues();
}
return new BytesValues.WithOrdinals(globalOrdinals) {
int readerIndex;
@Override
public BytesRef getValueByOrd(long globalOrd) {
final long segmentOrd = globalOrd - globalOrdToFirstSegmentDelta.get(globalOrd);
readerIndex = (int) globalOrdToFirstSegment.get(globalOrd);
return bytesValues[readerIndex].getValueByOrd(segmentOrd);
}
@Override
public BytesRef copyShared() {
return bytesValues[readerIndex].copyShared();
}
};
}
@Override
public boolean isMultiValued() {
return afd.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return afd.getNumberUniqueValues();
return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
}
@Override

View File

@ -19,15 +19,18 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
/**
* {@link Ordinals} implementation which is efficient at storing field data ordinals for multi-valued or sparse fields.
*/
public class MultiOrdinals implements Ordinals {
public class MultiOrdinals extends Ordinals {
private static final int OFFSETS_PAGE_SIZE = 1024;
private static final int OFFSET_INIT_PAGE_COUNT = 16;
@ -74,38 +77,36 @@ public class MultiOrdinals implements Ordinals {
assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds();
}
@Override
public long getMemorySizeInBytes() {
return endOffsets.ramBytesUsed() + ords.ramBytesUsed();
}
@Override
public boolean isMultiValued() {
return multiValued;
public WithOrdinals ordinals(ValuesHolder values) {
return new MultiDocs(this, values);
}
@Override
public long getMaxOrd() {
return maxOrd;
}
@Override
public Ordinals.Docs ordinals() {
return new MultiDocs(this);
}
static class MultiDocs extends Ordinals.AbstractDocs {
public static class MultiDocs extends BytesValues.WithOrdinals {
private final long maxOrd;
private final MonotonicAppendingLongBuffer endOffsets;
private final AppendingPackedLongBuffer ords;
private long offset;
private long limit;
private long currentOrd;
private final ValuesHolder values;
MultiDocs(MultiOrdinals ordinals) {
super(ordinals);
MultiDocs(MultiOrdinals ordinals, ValuesHolder values) {
super(ordinals.multiValued);
this.maxOrd = ordinals.maxOrd;
this.endOffsets = ordinals.endOffsets;
this.ords = ordinals.ords;
this.values = values;
}
@Override
public long getMaxOrd() {
return maxOrd;
}
@Override
@ -113,7 +114,7 @@ public class MultiOrdinals implements Ordinals {
final long startOffset = docId > 0 ? endOffsets.get(docId - 1) : 0;
final long endOffset = endOffsets.get(docId);
if (startOffset == endOffset) {
return currentOrd = Ordinals.MISSING_ORDINAL; // ord for missing values
return currentOrd = MISSING_ORDINAL; // ord for missing values
} else {
return currentOrd = ords.get(startOffset);
}
@ -138,5 +139,15 @@ public class MultiOrdinals implements Ordinals {
public long currentOrd() {
return currentOrd;
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.getValueByOrd(ord);
}
@Override
public BytesRef copyShared() {
return values.copy(scratch);
}
}
}

View File

@ -19,124 +19,37 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
/**
* A thread safe ordinals abstraction. Ordinals can only be positive integers.
*/
public interface Ordinals {
public abstract class Ordinals {
static final long MISSING_ORDINAL = SortedSetDocValues.NO_MORE_ORDS;
static final long MIN_ORDINAL = 0;
public static final ValuesHolder NO_VALUES = new ValuesHolder() {
@Override
public BytesRef getValueByOrd(long ord) {
throw new UnsupportedOperationException();
}
};
/**
* The memory size this ordinals take.
*/
long getMemorySizeInBytes();
public abstract long getMemorySizeInBytes();
/**
* Is one of the docs maps to more than one ordinal?
*/
boolean isMultiValued();
public abstract BytesValues.WithOrdinals ordinals(ValuesHolder values);
/**
* Returns total unique ord count; this includes +1 for
* the {@link #MISSING_ORDINAL} ord (always {@value #MISSING_ORDINAL} ).
*/
long getMaxOrd();
/**
* Returns a lightweight (non thread safe) view iterator of the ordinals.
*/
Docs ordinals();
/**
* A non thread safe ordinals abstraction, yet very lightweight to create. The idea
* is that this gets created for each "iteration" over ordinals.
* <p/>
* <p>A value of 0 ordinal when iterating indicated "no" value.</p>
* To iterate of a set of ordinals for a given document use {@link #setDocument(int)} and {@link #nextOrd()} as
* show in the example below:
* <pre>
* Ordinals.Docs docs = ...;
* final int len = docs.setDocId(docId);
* for (int i = 0; i < len; i++) {
* final long ord = docs.nextOrd();
* // process ord
* }
* </pre>
*/
interface Docs {
/**
* Returns total unique ord count; this includes +1 for
* the null ord (always 0).
*/
long getMaxOrd();
/**
* Is one of the docs maps to more than one ordinal?
*/
boolean isMultiValued();
/**
* The ordinal that maps to the relevant docId. If it has no value, returns
* <tt>0</tt>.
*/
long getOrd(int docId);
/**
* Returns the next ordinal for the current docID set to {@link #setDocument(int)}.
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
* is undefined.
*
* Note: This method will never return <tt>0</tt>.
*
* @return the next ordinal for the current docID set to {@link #setDocument(int)}.
*/
long nextOrd();
/**
* Sets iteration to the specified docID and returns the number of
* ordinals for this document ID,
* @param docId document ID
*
* @see #nextOrd()
*/
int setDocument(int docId);
/**
* Returns the current ordinal in the iteration
* @return the current ordinal in the iteration
*/
long currentOrd();
public final BytesValues.WithOrdinals ordinals() {
return ordinals(NO_VALUES);
}
/**
* Base implementation of {@link Docs}.
*/
public static abstract class AbstractDocs implements Docs {
public static interface ValuesHolder {
protected final Ordinals ordinals;
public AbstractDocs(Ordinals ordinals) {
this.ordinals = ordinals;
}
@Override
public final long getMaxOrd() {
return ordinals.getMaxOrd();
}
@Override
public final boolean isMultiValued() {
return ordinals.isMultiValued();
}
public abstract BytesRef getValueByOrd(long ord);
}

View File

@ -27,6 +27,7 @@ import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.BytesValues;
import java.io.Closeable;
import java.io.IOException;
@ -259,7 +260,7 @@ public final class OrdinalsBuilder implements Closeable {
}
private final int maxDoc;
private long currentOrd = Ordinals.MIN_ORDINAL - 1;
private long currentOrd = BytesValues.WithOrdinals.MIN_ORDINAL - 1;
private int numDocsWithValue = 0;
private int numMultiValuedDocs = 0;
private int totalNumOrds = 0;

View File

@ -19,12 +19,15 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
/**
*/
public class SinglePackedOrdinals implements Ordinals {
public class SinglePackedOrdinals extends Ordinals {
// ordinals with value 0 indicates no value
private final PackedInts.Reader reader;
@ -36,7 +39,7 @@ public class SinglePackedOrdinals implements Ordinals {
assert builder.getNumMultiValuesDocs() == 0;
this.maxOrd = builder.getMaxOrd();
// We don't reuse the builder as-is because it might have been built with a higher overhead ratio
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(getMaxOrd()), acceptableOverheadRatio);
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(maxOrd), acceptableOverheadRatio);
PackedInts.copy(builder.getFirstOrdinals(), 0, reader, 0, builder.maxDoc(), 8 * 1024);
this.reader = reader;
}
@ -50,29 +53,28 @@ public class SinglePackedOrdinals implements Ordinals {
}
@Override
public boolean isMultiValued() {
return false;
public WithOrdinals ordinals(ValuesHolder values) {
return new Docs(this, values);
}
@Override
public long getMaxOrd() {
return maxOrd;
}
@Override
public Docs ordinals() {
return new Docs(this, reader);
}
public static class Docs extends Ordinals.AbstractDocs {
private static class Docs extends BytesValues.WithOrdinals {
private final long maxOrd;
private final PackedInts.Reader reader;
private final ValuesHolder values;
private long currentOrdinal;
public Docs(SinglePackedOrdinals parent, PackedInts.Reader reader) {
super(parent);
this.reader = reader;
public Docs(SinglePackedOrdinals parent, ValuesHolder values) {
super(false);
this.maxOrd = parent.maxOrd;
this.reader = parent.reader;
this.values = values;
}
@Override
public long getMaxOrd() {
return maxOrd;
}
@Override
@ -82,7 +84,7 @@ public class SinglePackedOrdinals implements Ordinals {
@Override
public long nextOrd() {
assert currentOrdinal >= Ordinals.MIN_ORDINAL;
assert currentOrdinal >= MIN_ORDINAL;
return currentOrdinal;
}
@ -97,5 +99,15 @@ public class SinglePackedOrdinals implements Ordinals {
public long currentOrd() {
return currentOrdinal;
}
@Override
public BytesRef getValueByOrd(long ord) {
return values.getValueByOrd(ord);
}
@Override
public BytesRef copyShared() {
return values.copy(scratch);
}
}
}

View File

@ -38,16 +38,6 @@ abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<Ato
protected static class Empty extends AtomicGeoPointFieldData<ScriptDocValues> {
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return 0;
}
@Override
public long getMemorySizeInBytes() {
return 0;

View File

@ -26,7 +26,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import java.io.IOException;
import java.util.Comparator;
@ -38,16 +37,14 @@ import java.util.Comparator;
public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
private final BytesValues.WithOrdinals bytesValues;
private final Ordinals.Docs ordinals;
private final long maxOrd;
private long currentOrd = Ordinals.MISSING_ORDINAL;
private long currentOrd = BytesValues.WithOrdinals.MISSING_ORDINAL;
private BytesRef currentTerm;
public AtomicFieldDataWithOrdinalsTermsEnum(AtomicFieldData.WithOrdinals afd) {
this.bytesValues = afd.getBytesValues();
this.ordinals = bytesValues.ordinals();
this.maxOrd = ordinals.getMaxOrd();
this.maxOrd = bytesValues.getMaxOrd();
}
@Override
@ -70,9 +67,9 @@ public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
@Override
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord < ordinals.getMaxOrd();
assert ord >= 0 && ord < bytesValues.getMaxOrd();
currentOrd = ord;
if (currentOrd == Ordinals.MISSING_ORDINAL) {
if (currentOrd == BytesValues.WithOrdinals.MISSING_ORDINAL) {
currentTerm = null;
} else {
currentTerm = bytesValues.getValueByOrd(currentOrd);
@ -125,7 +122,7 @@ public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
final private static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
long low = 1;
long high = a.ordinals().getMaxOrd();
long high = a.getMaxOrd();
while (low <= high) {
long mid = (low + high) >>> 1;
BytesRef midVal = a.getValueByOrd(mid);

View File

@ -43,17 +43,6 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
this.field = field;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
// probably not accurate, but a good upper limit
return reader.maxDoc();
}
@Override
public long getMemorySizeInBytes() {
// TODO: Lucene doesn't expose it right now
@ -81,6 +70,9 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
return new BytesValues(false) {
final BytesRef scratch = new BytesRef();
int docId;
@Override
public int setDocument(int docId) {
this.docId = docId;

View File

@ -137,16 +137,6 @@ final class BinaryDVNumericAtomicFieldData extends AbstractAtomicNumericFieldDat
}
}
@Override
public boolean isMultiValued() {
return true; // no way to know
}
@Override
public long getNumberUniqueValues() {
return Long.MAX_VALUE; // no clue
}
@Override
public long getMemorySizeInBytes() {
return -1; // Lucene doesn't expose it

View File

@ -36,16 +36,6 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocVal
this.values = values == null ? DocValues.EMPTY_BINARY : values;
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public long getNumberUniqueValues() {
return Long.MAX_VALUE;
}
@Override
public long getMemorySizeInBytes() {
return -1; // not exposed by Lucene
@ -56,6 +46,7 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocVal
return new BytesValues(true) {
final BytesRef bytes = new BytesRef();
final BytesRef scratch = new BytesRef();
final ByteArrayDataInput in = new ByteArrayDataInput();
@Override

View File

@ -59,16 +59,6 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
return DoubleValues.EMPTY;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return 0;
}
@Override
public long getMemorySizeInBytes() {
return 0;
@ -96,16 +86,6 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -130,14 +110,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
private final DoubleArray values;
LongValues(DoubleArray values, Ordinals.Docs ordinals) {
LongValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public final long getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return (long) values.get(ord);
}
}
@ -146,14 +126,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
private final DoubleArray values;
DoubleValues(DoubleArray values, Ordinals.Docs ordinals) {
DoubleValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public double getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
}
@ -167,23 +147,11 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
private final DoubleArray values;
private final FixedBitSet set;
private final long numOrds;
public SingleFixedSet(DoubleArray values, FixedBitSet set, long numOrds) {
public SingleFixedSet(DoubleArray values, FixedBitSet set) {
super();
this.values = values;
this.set = set;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -257,26 +225,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
public static class Single extends DoubleArrayAtomicFieldData {
private final DoubleArray values;
private final long numOrds;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(DoubleArray values, long numOrds) {
public Single(DoubleArray values) {
super();
this.values = values;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override

View File

@ -32,7 +32,6 @@ import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -101,10 +100,10 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
}
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new DoubleArrayAtomicFieldData.WithOrdinals(values, build);
} else {
Docs ordinals = build.ordinals();
final FixedBitSet set = builder.buildDocsWithValuesSet();
// there's sweet spot where due to low unique value count, using ordinals will consume less memory
@ -121,15 +120,15 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
DoubleArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(maxDoc);
for (int i = 0; i < maxDoc; i++) {
final long ordinal = ordinals.getOrd(i);
if (ordinal != Ordinals.MISSING_ORDINAL) {
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
sValues.set(i, values.get(ordinal));
}
}
assert sValues.size() == maxDoc;
if (set == null) {
data = new DoubleArrayAtomicFieldData.Single(sValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new DoubleArrayAtomicFieldData.Single(sValues);
} else {
data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, set);
}
}
success = true;

View File

@ -1,51 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
* An empty {@link org.elasticsearch.index.fielddata.BytesValues.WithOrdinals} implementation
*/
final class EmptyByteValuesWithOrdinals extends BytesValues.WithOrdinals {
EmptyByteValuesWithOrdinals(Ordinals.Docs ordinals) {
super(ordinals);
}
@Override
public BytesRef getValueByOrd(long ord) {
scratch.length = 0;
return scratch;
}
@Override
public int setDocument(int docId) {
return 0;
}
@Override
public BytesRef nextValue() {
throw new ElasticsearchIllegalStateException("Empty BytesValues has no next value");
}
}

View File

@ -26,8 +26,8 @@ import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
import org.apache.lucene.util.fst.Util;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import java.io.IOException;
@ -36,10 +36,6 @@ import java.io.IOException;
*/
public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public static FSTBytesAtomicFieldData empty() {
return new Empty();
}
// 0 ordinal in values means no value (its null)
protected final Ordinals ordinals;
@ -56,16 +52,6 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
public void close() {
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -79,8 +65,7 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
@Override
public BytesValues.WithOrdinals getBytesValues() {
assert fst != null;
return new BytesValues(fst, ordinals.ordinals());
return ordinals.ordinals(new ValuesHolder(fst));
}
@Override
@ -89,32 +74,26 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public TermsEnum getTermsEnum() {
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
}
private static class ValuesHolder implements Ordinals.ValuesHolder {
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
protected final FST<Long> fst;
protected final Ordinals.Docs ordinals;
private final FST<Long> fst;
// per-thread resources
private final BytesRef scratch;
protected final BytesReader in;
protected final Arc<Long> firstArc = new Arc<>();
protected final Arc<Long> scratchArc = new Arc<>();
protected final IntsRef scratchInts = new IntsRef();
BytesValues(FST<Long> fst, Ordinals.Docs ordinals) {
super(ordinals);
ValuesHolder(FST<Long> fst) {
this.fst = fst;
this.ordinals = ordinals;
scratch = new BytesRef();
in = fst.getBytesReader();
}
@Override
public BytesRef getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
in.setPosition(0);
fst.getFirstArc(firstArc);
try {
@ -127,30 +106,6 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
}
return scratch;
}
}
final static class Empty extends FSTBytesAtomicFieldData {
Empty() {
super(null, EmptyOrdinals.INSTANCE);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
}
@Override
public ScriptDocValues.Strings getScriptValues() {
return ScriptDocValues.EMPTY_STRINGS;
}
}
}

View File

@ -27,9 +27,7 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -40,14 +38,14 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
/**
*/
public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytesAtomicFieldData> {
public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
private final CircuitBreakerService breakerService;
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<FSTBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
GlobalOrdinalsBuilder globalOrdinalBuilder) {
return new FSTBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
@ -61,7 +59,7 @@ public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytes
}
@Override
public FSTBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
@ -69,9 +67,8 @@ public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytes
// TODO: Use an actual estimator to estimate before loading.
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
if (terms == null) {
data = FSTBytesAtomicFieldData.empty();
estimator.afterLoad(null, data.getMemorySizeInBytes());
return data;
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.getMemorySizeInBytes());
return AtomicFieldData.WithOrdinals.EMPTY;
}
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
org.apache.lucene.util.fst.Builder<Long> fstBuilder = new org.apache.lucene.util.fst.Builder<>(INPUT_TYPE.BYTE1, outputs);

View File

@ -58,16 +58,6 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
return DoubleValues.EMPTY;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return 0;
}
@Override
public long getMemorySizeInBytes() {
return 0;
@ -95,16 +85,6 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -127,14 +107,14 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
private final FloatArray values;
LongValues(FloatArray values, Ordinals.Docs ordinals) {
LongValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public long getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return (long) values.get(ord);
}
}
@ -143,7 +123,7 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
private final FloatArray values;
DoubleValues(FloatArray values, Ordinals.Docs ordinals) {
DoubleValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@ -163,23 +143,11 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
private final FloatArray values;
private final FixedBitSet set;
private final long numOrd;
public SingleFixedSet(FloatArray values, FixedBitSet set, long numOrd) {
public SingleFixedSet(FloatArray values, FixedBitSet set) {
super();
this.values = values;
this.set = set;
this.numOrd = numOrd;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrd;
}
@Override
@ -255,26 +223,14 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
public static class Single extends FloatArrayAtomicFieldData {
private final FloatArray values;
private final long numOrd;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(FloatArray values, long numOrd) {
public Single(FloatArray values) {
super();
this.values = values;
this.numOrd = numOrd;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrd;
}
@Override

View File

@ -31,7 +31,6 @@ import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -99,10 +98,10 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
}
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new FloatArrayAtomicFieldData.WithOrdinals(values, build);
} else {
Docs ordinals = build.ordinals();
final FixedBitSet set = builder.buildDocsWithValuesSet();
// there's sweet spot where due to low unique value count, using ordinals will consume less memory
@ -119,15 +118,15 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
FloatArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(maxDoc);
for (int i = 0; i < maxDoc; i++) {
final long ordinal = ordinals.getOrd(i);
if (ordinal != Ordinals.MISSING_ORDINAL) {
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
sValues.set(i, values.get(ordinal));
}
}
assert sValues.size() == maxDoc;
if (set == null) {
data = new FloatArrayAtomicFieldData.Single(sValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new FloatArrayAtomicFieldData.Single(sValues);
} else {
data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, set);
}
}
success = true;

View File

@ -37,16 +37,6 @@ final class GeoPointBinaryDVAtomicFieldData extends AtomicGeoPointFieldData<Scri
this.values = values == null ? DocValues.EMPTY_BINARY : values;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return Long.MAX_VALUE;
}
@Override
public long getMemorySizeInBytes() {
return -1; // not exposed by Lucene

View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PagedMutable;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
@ -58,16 +59,6 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -85,11 +76,11 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final Ordinals.Docs ordinals;
private final BytesValues.WithOrdinals ordinals;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesWithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, Ordinals.Docs ordinals) {
GeoPointValuesWithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.encoding = encoding;
this.lon = lon;
@ -119,25 +110,13 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final FixedBitSet set;
private final long numOrds;
public SingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set, long numOrds) {
public SingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set) {
super();
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
this.set = set;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -190,24 +169,12 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
private final GeoPointFieldMapper.Encoding encoding;
private final PagedMutable lon, lat;
private final long numOrds;
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, long numOrds) {
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat) {
super();
this.encoding = encoding;
this.lon = lon;
this.lat = lat;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override

View File

@ -33,7 +33,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -115,21 +114,21 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
}
Ordinals build = builder.build(fieldDataType.getSettings());
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
if (lat.size() != build.getMaxOrd()) {
lat = lat.resize(build.getMaxOrd());
lon = lon.resize(build.getMaxOrd());
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
if (lat.size() != ordinals.getMaxOrd()) {
lat = lat.resize(ordinals.getMaxOrd());
lon = lon.resize(ordinals.getMaxOrd());
}
data = new GeoPointCompressedAtomicFieldData.WithOrdinals(encoding, lon, lat, build);
} else {
Docs ordinals = build.ordinals();
int maxDoc = reader.maxDoc();
PagedMutable sLat = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
PagedMutable sLon = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
final long missing = encoding.encodeCoordinate(0);
for (int i = 0; i < maxDoc; i++) {
final long nativeOrdinal = ordinals.getOrd(i);
if (nativeOrdinal != Ordinals.MISSING_ORDINAL) {
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
sLat.set(i, lat.get(nativeOrdinal));
sLon.set(i, lon.get(nativeOrdinal));
} else {
@ -139,9 +138,9 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
}
FixedBitSet set = builder.buildDocsWithValuesSet();
if (set == null) {
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat);
} else {
data = new GeoPointCompressedAtomicFieldData.SingleFixedSet(encoding, sLon, sLat, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new GeoPointCompressedAtomicFieldData.SingleFixedSet(encoding, sLon, sLat, set);
}
}
success = true;

View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
@ -54,16 +55,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -80,11 +71,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
public static class GeoPointValuesWithOrdinals extends GeoPointValues {
private final DoubleArray lon, lat;
private final Ordinals.Docs ordinals;
private final BytesValues.WithOrdinals ordinals;
private final GeoPoint scratch = new GeoPoint();
GeoPointValuesWithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals.Docs ordinals) {
GeoPointValuesWithOrdinals(DoubleArray lon, DoubleArray lat, BytesValues.WithOrdinals ordinals) {
super(ordinals.isMultiValued());
this.lon = lon;
this.lat = lat;
@ -112,24 +103,12 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
private final DoubleArray lon, lat;
private final FixedBitSet set;
private final long numOrds;
public SingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set, long numOrds) {
public SingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set) {
super();
this.lon = lon;
this.lat = lat;
this.set = set;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -180,23 +159,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
public static class Single extends GeoPointDoubleArrayAtomicFieldData {
private final DoubleArray lon, lat;
private final long numOrds;
public Single(DoubleArray lon, DoubleArray lat, long numOrds) {
public Single(DoubleArray lon, DoubleArray lat) {
super();
this.lon = lon;
this.lat = lat;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override

View File

@ -30,7 +30,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
@ -90,23 +89,23 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms);
Ordinals build = builder.build(fieldDataType.getSettings());
if (!(build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
Docs ordinals = build.ordinals();
BytesValues.WithOrdinals ordinals = build.ordinals();
if (!(ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
int maxDoc = reader.maxDoc();
DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
for (int i = 0; i < maxDoc; i++) {
long nativeOrdinal = ordinals.getOrd(i);
if (nativeOrdinal != Ordinals.MISSING_ORDINAL) {
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
sLat.set(i, lat.get(nativeOrdinal));
sLon.set(i, lon.get(nativeOrdinal));
}
}
FixedBitSet set = builder.buildDocsWithValuesSet();
if (set == null) {
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat);
} else {
data = new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, set);
}
} else {
data = new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon, lat, build);

View File

@ -28,7 +28,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
@ -46,7 +45,15 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
}
private static final Ordinals.Docs INDEX_ORDINALS = new Ordinals.Docs() {
private static class IndexBytesValues extends BytesValues.WithOrdinals {
private final BytesRef scratch;
protected IndexBytesValues(String index) {
super(false);
scratch = new BytesRef();
scratch.copyChars(index);
}
@Override
public int setDocument(int docId) {
@ -55,17 +62,12 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
@Override
public long nextOrd() {
return Ordinals.MIN_ORDINAL;
}
@Override
public boolean isMultiValued() {
return false;
return BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
public long getOrd(int docId) {
return Ordinals.MIN_ORDINAL;
return BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
@ -75,15 +77,7 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
@Override
public long currentOrd() {
return Ordinals.MIN_ORDINAL;
}
};
private static class IndexBytesValues extends BytesValues.WithOrdinals {
protected IndexBytesValues(String index) {
super(INDEX_ORDINALS);
scratch.copyChars(index);
return BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
@ -106,16 +100,6 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
return 0;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return 1;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new IndexBytesValues(index);

View File

@ -48,17 +48,6 @@ public class NumericDVAtomicFieldData extends AbstractAtomicNumericFieldData {
// no-op
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
// good upper limit
return reader.maxDoc();
}
@Override
public long getMemorySizeInBytes() {
// TODO: cannot be computed from Lucene

View File

@ -58,21 +58,11 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
return DoubleValues.EMPTY;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getMemorySizeInBytes() {
return 0;
}
@Override
public long getNumberUniqueValues() {
return 0;
}
@Override
public BytesValues getBytesValues() {
return BytesValues.EMPTY;
@ -95,11 +85,6 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -108,11 +93,6 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
return size;
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
@ -127,14 +107,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
private final MonotonicAppendingLongBuffer values;
LongValues(MonotonicAppendingLongBuffer values, Ordinals.Docs ordinals) {
LongValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public long getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
}
@ -143,14 +123,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
private final MonotonicAppendingLongBuffer values;
DoubleValues(MonotonicAppendingLongBuffer values, Ordinals.Docs ordinals) {
DoubleValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
super(ordinals);
this.values = values;
}
@Override
public double getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
return values.get(ord);
}
@ -167,24 +147,12 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
private final PackedInts.Mutable values;
private final long minValue;
private final long missingValue;
private final long numOrds;
public SingleSparse(PackedInts.Mutable values, long minValue, long missingValue, long numOrds) {
public SingleSparse(PackedInts.Mutable values, long minValue, long missingValue) {
super();
this.values = values;
this.minValue = minValue;
this.missingValue = missingValue;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -263,27 +231,15 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
private final PackedInts.Mutable values;
private final long minValue;
private final long numOrds;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(PackedInts.Mutable values, long minValue, long numOrds) {
public Single(PackedInts.Mutable values, long minValue) {
super();
this.values = values;
this.minValue = minValue;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -355,26 +311,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
public static class PagedSingle extends PackedArrayAtomicFieldData {
private final AppendingDeltaPackedLongBuffer values;
private final long numOrds;
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public PagedSingle(AppendingDeltaPackedLongBuffer values, long numOrds) {
public PagedSingle(AppendingDeltaPackedLongBuffer values) {
super();
this.values = values;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override
@ -446,23 +390,11 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
private final AppendingDeltaPackedLongBuffer values;
private final FixedBitSet docsWithValue;
private final long numOrds;
public PagedSingleSparse(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue, long numOrds) {
public PagedSingleSparse(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue) {
super();
this.values = values;
this.docsWithValue = docsWithValue;
this.numOrds = numOrds;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return numOrds;
}
@Override

View File

@ -35,15 +35,14 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
import java.util.EnumSet;
@ -128,10 +127,10 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
Ordinals build = builder.build(fieldDataType.getSettings());
CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType);
if (build.isMultiValued() || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
BytesValues.WithOrdinals ordinals = build.ordinals();
if (ordinals.isMultiValued() || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
data = new PackedArrayAtomicFieldData.WithOrdinals(values, build);
} else {
Docs ordinals = build.ordinals();
final FixedBitSet docsWithValues = builder.buildDocsWithValuesSet();
long minValue, maxValue;
@ -185,15 +184,15 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
for (int i = 0; i < reader.maxDoc(); i++) {
final long ord = ordinals.getOrd(i);
if (ord != Ordinals.MISSING_ORDINAL) {
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
long value = values.get(ord);
sValues.set(i, value - minValue);
}
}
if (docsWithValues == null) {
data = new PackedArrayAtomicFieldData.Single(sValues, minValue, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new PackedArrayAtomicFieldData.Single(sValues, minValue);
} else {
data = new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, missingValue, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, missingValue);
}
break;
case PAGED:
@ -203,16 +202,16 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
long lastValue = 0;
for (int i = 0; i < reader.maxDoc(); i++) {
final long ord = ordinals.getOrd(i);
if (ord != Ordinals.MISSING_ORDINAL) {
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
lastValue = values.get(ord);
}
dpValues.add(lastValue);
}
dpValues.freeze();
if (docsWithValues == null) {
data = new PackedArrayAtomicFieldData.PagedSingle(dpValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new PackedArrayAtomicFieldData.PagedSingle(dpValues);
} else {
data = new PackedArrayAtomicFieldData.PagedSingleSparse(dpValues, docsWithValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
data = new PackedArrayAtomicFieldData.PagedSingleSparse(dpValues, docsWithValues);
}
break;
case ORDINALS:
@ -239,7 +238,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
}
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, Docs ordinals,
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, BytesValues.WithOrdinals ordinals,
long minValue, long maxValue, float acceptableOverheadRatio, int pageSize) {
CommonSettings.MemoryStorageFormat format;
@ -261,7 +260,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
long pageMaxOrdinal = Long.MIN_VALUE;
for (int i = 1; i < reader.maxDoc(); ++i, pageIndex = (pageIndex + 1) % pageSize) {
long ordinal = ordinals.getOrd(i);
if (ordinal != Ordinals.MISSING_ORDINAL) {
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
pageMaxOrdinal = Math.max(ordinal, pageMaxOrdinal);
pageMinOrdinal = Math.min(ordinal, pageMinOrdinal);
}

View File

@ -23,18 +23,14 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public static PagedBytesAtomicFieldData empty() {
return new Empty();
}
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
protected final Ordinals ordinals;
@ -53,16 +49,6 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
public void close() {
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public long getNumberUniqueValues() {
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
@ -78,7 +64,7 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new BytesValues(bytes, termOrdToBytesOffset, ordinals.ordinals());
return ordinals.ordinals(new ValuesHolder(bytes, termOrdToBytesOffset));
}
@Override
@ -86,82 +72,24 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public TermsEnum getTermsEnum() {
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
}
private static class ValuesHolder implements Ordinals.ValuesHolder {
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
private final BytesRef scratch = new BytesRef();
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
protected final PagedBytes.Reader bytes;
protected final MonotonicAppendingLongBuffer termOrdToBytesOffset;
protected final Ordinals.Docs ordinals;
BytesValues(PagedBytes.Reader bytes, MonotonicAppendingLongBuffer termOrdToBytesOffset, Ordinals.Docs ordinals) {
super(ordinals);
ValuesHolder(PagedBytes.Reader bytes, MonotonicAppendingLongBuffer termOrdToBytesOffset) {
this.bytes = bytes;
this.termOrdToBytesOffset = termOrdToBytesOffset;
this.ordinals = ordinals;
}
@Override
public BytesRef copyShared() {
// when we fill from the pages bytes, we just reference an existing buffer slice, its enough
// to create a shallow copy of the bytes to be safe for "reads".
return new BytesRef(scratch.bytes, scratch.offset, scratch.length);
}
@Override
public final Ordinals.Docs ordinals() {
return this.ordinals;
}
@Override
public final BytesRef getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
public BytesRef getValueByOrd(long ord) {
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
bytes.fill(scratch, termOrdToBytesOffset.get(ord));
return scratch;
}
@Override
public final BytesRef nextValue() {
bytes.fill(scratch, termOrdToBytesOffset.get(ordinals.nextOrd()));
return scratch;
}
}
private final static class Empty extends PagedBytesAtomicFieldData {
Empty() {
super(emptyBytes(), 0, new MonotonicAppendingLongBuffer(), EmptyOrdinals.INSTANCE);
}
static PagedBytes.Reader emptyBytes() {
PagedBytes bytes = new PagedBytes(1);
bytes.copyUsingLengthPrefix(new BytesRef());
return bytes.freeze(true);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public long getNumberUniqueValues() {
return 0;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
}
@Override
public ScriptDocValues.Strings getScriptValues() {
return ScriptDocValues.EMPTY_STRINGS;
}
}
}

View File

@ -26,10 +26,7 @@ import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -42,13 +39,13 @@ import java.io.IOException;
/**
*/
public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<PagedBytesAtomicFieldData> {
public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<PagedBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
GlobalOrdinalsBuilder globalOrdinalBuilder) {
return new PagedBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
@ -62,15 +59,14 @@ public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<PagedB
}
@Override
public PagedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(), getFieldNames().fullName());
Terms terms = reader.terms(getFieldNames().indexName());
if (terms == null) {
PagedBytesAtomicFieldData emptyData = PagedBytesAtomicFieldData.empty();
estimator.adjustForNoTerms(emptyData.getMemorySizeInBytes());
return emptyData;
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.getMemorySizeInBytes());
return AtomicFieldData.WithOrdinals.EMPTY;
}
final PagedBytes bytes = new PagedBytes(15);

View File

@ -31,16 +31,10 @@ import org.elasticsearch.index.fielddata.ScriptDocValues;
public class ParentChildAtomicFieldData implements AtomicFieldData {
private final ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds;
private final long numberUniqueValues;
private final long memorySizeInBytes;
public ParentChildAtomicFieldData(ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds) {
this.typeToIds = typeToIds;
long numValues = 0;
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
numValues += cursor.value.getNumberUniqueValues();
}
this.numberUniqueValues = numValues;
long size = 0;
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
size += cursor.value.getMemorySizeInBytes();
@ -48,16 +42,6 @@ public class ParentChildAtomicFieldData implements AtomicFieldData {
this.memorySizeInBytes = size;
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public long getNumberUniqueValues() {
return numberUniqueValues;
}
@Override
public long getMemorySizeInBytes() {
return memorySizeInBytes;
@ -72,6 +56,7 @@ public class ParentChildAtomicFieldData implements AtomicFieldData {
}
return new BytesValues(true) {
private final BytesRef scratch = new BytesRef();
private final BytesRef[] terms = new BytesRef[2];
private int index;

View File

@ -367,7 +367,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
if(typeAfd != null) {
return typeAfd;
} else {
return PagedBytesAtomicFieldData.empty();
return AtomicFieldData.WithOrdinals.EMPTY;
}
}

View File

@ -28,7 +28,6 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import java.io.IOException;
@ -70,7 +69,7 @@ abstract class SortedSetDVAtomicFieldData {
public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getBytesValues() {
final SortedSetDocValues values = getValuesNoException(reader, field);
return new SortedSetValues(reader, field, values);
return new SortedSetValues(values);
}
public TermsEnum getTermsEnum() {
@ -93,79 +92,23 @@ abstract class SortedSetDVAtomicFieldData {
static class SortedSetValues extends BytesValues.WithOrdinals {
protected final SortedSetDocValues values;
SortedSetValues(AtomicReader reader, String field, SortedSetDocValues values) {
super(new SortedSetDocs(new SortedSetOrdinals(reader, field, values.getValueCount()), values));
this.values = values;
}
@Override
public BytesRef getValueByOrd(long ord) {
assert ord != Ordinals.MISSING_ORDINAL;
values.lookupOrd(ord, scratch);
return scratch;
}
@Override
public BytesRef nextValue() {
values.lookupOrd(ordinals.nextOrd(), scratch);
return scratch;
}
}
static final class SortedSetOrdinals implements Ordinals {
// We don't store SortedSetDocValues as a member because Ordinals must be thread-safe
private final AtomicReader reader;
private final String field;
private final long maxOrd;
public SortedSetOrdinals(AtomicReader reader, String field, long numOrds) {
super();
this.reader = reader;
this.field = field;
this.maxOrd = numOrds;
}
@Override
public long getMemorySizeInBytes() {
// Ordinals can't be distinguished from the atomic field data instance
return -1;
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public long getMaxOrd() {
return maxOrd;
}
@Override
public Docs ordinals() {
final SortedSetDocValues values = getValuesNoException(reader, field);
assert values.getValueCount() == maxOrd;
return new SortedSetDocs(this, values);
}
}
static class SortedSetDocs extends Ordinals.AbstractDocs {
private final BytesRef scratch = new BytesRef();
private final SortedSetDocValues values;
private long[] ords;
private int ordIndex = Integer.MAX_VALUE;
private long currentOrdinal = -1;
SortedSetDocs(SortedSetOrdinals ordinals, SortedSetDocValues values) {
super(ordinals);
SortedSetValues(SortedSetDocValues values) {
super(DocValues.unwrapSingleton(values) == null);
this.values = values;
ords = new long[0];
}
@Override
public long getMaxOrd() {
return values.getValueCount();
}
@Override
public long getOrd(int docId) {
values.setDocument(docId);
@ -196,5 +139,11 @@ abstract class SortedSetDVAtomicFieldData {
public long currentOrd() {
return currentOrdinal;
}
@Override
public BytesRef getValueByOrd(long ord) {
values.lookupOrd(ord, scratch);
return scratch;
}
}
}

View File

@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.search.internal.SearchContext;
@ -107,8 +106,7 @@ public class ChildrenConstantScoreQuery extends Query {
} else {
AtomicFieldData.WithOrdinals afd = globalIfd.load(leaves.get(0));
BytesValues.WithOrdinals globalValues = afd.getBytesValues();
Ordinals.Docs globalOrdinals = globalValues.ordinals();
maxOrd = globalOrdinals.getMaxOrd();
maxOrd = globalValues.getMaxOrd();
}
if (maxOrd == 0) {
@ -242,8 +240,7 @@ public class ChildrenConstantScoreQuery extends Query {
LongBitSet parentOrds = collector.parentOrds;
BytesValues.WithOrdinals globalValues = globalIfd.load(context).getBytesValues();
if (globalValues != null) {
Ordinals.Docs globalOrdinals = globalValues.ordinals();
DocIdSetIterator parentIdIterator = new ParentOrdIterator(innerIterator, parentOrds, globalOrdinals, this);
DocIdSetIterator parentIdIterator = new ParentOrdIterator(innerIterator, parentOrds, globalValues, this);
return ConstantScorer.create(parentIdIterator, this, queryWeight);
}
}
@ -259,7 +256,6 @@ public class ChildrenConstantScoreQuery extends Query {
private final ParentChildIndexFieldData.WithOrdinals indexFieldData;
private BytesValues.WithOrdinals values;
private Ordinals.Docs globalOrdinals;
private ParentOrdCollector(ParentChildIndexFieldData.WithOrdinals indexFieldData, long maxOrd) {
// TODO: look into reusing LongBitSet#bits array
@ -269,9 +265,9 @@ public class ChildrenConstantScoreQuery extends Query {
@Override
public void collect(int doc) throws IOException {
if (globalOrdinals != null) {
long globalOrdinal = globalOrdinals.getOrd(doc);
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
if (values != null) {
long globalOrdinal = values.getOrd(doc);
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
parentOrds.set(globalOrdinal);
}
}
@ -280,11 +276,6 @@ public class ChildrenConstantScoreQuery extends Query {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
values = indexFieldData.load(context).getBytesValues();
if (values != null) {
globalOrdinals = values.ordinals();
} else {
globalOrdinals = null;
}
}
long foundParents() {
@ -296,10 +287,10 @@ public class ChildrenConstantScoreQuery extends Query {
private final static class ParentOrdIterator extends FilteredDocIdSetIterator {
private final LongBitSet parentOrds;
private final Ordinals.Docs ordinals;
private final BytesValues.WithOrdinals ordinals;
private final ParentWeight parentWeight;
private ParentOrdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, Ordinals.Docs ordinals, ParentWeight parentWeight) {
private ParentOrdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, BytesValues.WithOrdinals ordinals, ParentWeight parentWeight) {
super(innerIterator);
this.parentOrds = parentOrds;
this.ordinals = ordinals;
@ -318,7 +309,7 @@ public class ChildrenConstantScoreQuery extends Query {
}
long parentOrd = ordinals.getOrd(doc);
if (parentOrd != Ordinals.MISSING_ORDINAL) {
if (parentOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
boolean match = parentOrds.get(parentOrd);
if (match) {
parentWeight.remaining--;

View File

@ -37,7 +37,6 @@ import org.elasticsearch.common.util.IntArray;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.internal.SearchContext.Lifetime;
@ -289,25 +288,23 @@ public class ChildrenQuery extends Query {
return null;
}
Ordinals.Docs globalOrdinals = bytesValues.ordinals();
if (minChildren > 0 || maxChildren != 0 || scoreType == ScoreType.NONE) {
switch (scoreType) {
case NONE:
DocIdSetIterator parentIdIterator = new CountParentOrdIterator(this, parents, collector, globalOrdinals,
DocIdSetIterator parentIdIterator = new CountParentOrdIterator(this, parents, collector, bytesValues,
minChildren, maxChildren);
return ConstantScorer.create(parentIdIterator, this, queryWeight);
case AVG:
return new AvgParentCountScorer(this, parents, collector, globalOrdinals, minChildren, maxChildren);
return new AvgParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
default:
return new ParentCountScorer(this, parents, collector, globalOrdinals, minChildren, maxChildren);
return new ParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
}
}
switch (scoreType) {
case AVG:
return new AvgParentScorer(this, parents, collector, globalOrdinals);
return new AvgParentScorer(this, parents, collector, bytesValues);
default:
return new ParentScorer(this, parents, collector, globalOrdinals);
return new ParentScorer(this, parents, collector, bytesValues);
}
}
return null;
@ -321,7 +318,6 @@ public class ChildrenQuery extends Query {
protected final BigArrays bigArrays;
protected final SearchContext searchContext;
protected Ordinals.Docs globalOrdinals;
protected BytesValues.WithOrdinals values;
protected Scorer scorer;
@ -334,9 +330,9 @@ public class ChildrenQuery extends Query {
@Override
public final void collect(int doc) throws IOException {
if (globalOrdinals != null) {
final long globalOrdinal = globalOrdinals.getOrd(doc);
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
if (values != null) {
final long globalOrdinal = values.getOrd(doc);
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
long parentIdx = parentIdxs.add(globalOrdinal);
if (parentIdx >= 0) {
newParent(parentIdx);
@ -361,9 +357,6 @@ public class ChildrenQuery extends Query {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
values = globalIfd.load(context).getBytesValues();
if (values != null) {
globalOrdinals = values.ordinals();
}
}
@Override
@ -507,13 +500,13 @@ public class ChildrenQuery extends Query {
final LongHash parentIds;
final FloatArray scores;
final Ordinals.Docs globalOrdinals;
final BytesValues.WithOrdinals globalOrdinals;
final DocIdSetIterator parentsIterator;
int currentDocId = -1;
float currentScore;
ParentScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals) {
ParentScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals) {
super(parentWeight);
this.parentWeight = parentWeight;
this.globalOrdinals = globalOrdinals;
@ -557,7 +550,7 @@ public class ChildrenQuery extends Query {
}
final long globalOrdinal = globalOrdinals.getOrd(currentDocId);
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
continue;
}
@ -583,7 +576,7 @@ public class ChildrenQuery extends Query {
}
final long globalOrdinal = globalOrdinals.getOrd(currentDocId);
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
return nextDoc();
}
@ -609,7 +602,7 @@ public class ChildrenQuery extends Query {
protected final int minChildren;
protected final int maxChildren;
ParentCountScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals, int minChildren, int maxChildren) {
ParentCountScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals, int minChildren, int maxChildren) {
super(parentWeight, parentsIterator, (ParentScoreCollector) collector, globalOrdinals);
this.minChildren = minChildren;
this.maxChildren = maxChildren == 0 ? Integer.MAX_VALUE : maxChildren;
@ -627,7 +620,7 @@ public class ChildrenQuery extends Query {
private static final class AvgParentScorer extends ParentCountScorer {
AvgParentScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals) {
AvgParentScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals) {
super(weight, parentsIterator, collector, globalOrdinals, 0, 0);
}
@ -642,7 +635,7 @@ public class ChildrenQuery extends Query {
private static final class AvgParentCountScorer extends ParentCountScorer {
AvgParentCountScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals, int minChildren, int maxChildren) {
AvgParentCountScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals, int minChildren, int maxChildren) {
super(weight, parentsIterator, collector, globalOrdinals, minChildren, maxChildren);
}
@ -664,10 +657,10 @@ public class ChildrenQuery extends Query {
protected final IntArray occurrences;
private final int minChildren;
private final int maxChildren;
private final Ordinals.Docs ordinals;
private final BytesValues.WithOrdinals ordinals;
private final ParentWeight parentWeight;
private CountParentOrdIterator(ParentWeight parentWeight, DocIdSetIterator innerIterator, ParentCollector collector, Ordinals.Docs ordinals, int minChildren, int maxChildren) {
private CountParentOrdIterator(ParentWeight parentWeight, DocIdSetIterator innerIterator, ParentCollector collector, BytesValues.WithOrdinals ordinals, int minChildren, int maxChildren) {
super(innerIterator);
this.parentIds = ((CountCollector) collector).parentIdxs;
this.occurrences = ((CountCollector) collector).occurrences;
@ -689,7 +682,7 @@ public class ChildrenQuery extends Query {
}
final long parentOrd = ordinals.getOrd(doc);
if (parentOrd != Ordinals.MISSING_ORDINAL) {
if (parentOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
final long parentIdx = parentIds.find(parentOrd);
if (parentIdx != -1) {
parentWeight.remaining--;

View File

@ -31,7 +31,6 @@ import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import java.io.IOException;
@ -96,8 +95,7 @@ public class ParentConstantScoreQuery extends Query {
} else {
AtomicFieldData.WithOrdinals afd = globalIfd.load(leaves.get(0));
BytesValues.WithOrdinals globalValues = afd.getBytesValues();
Ordinals.Docs globalOrdinals = globalValues.ordinals();
maxOrd = globalOrdinals.getMaxOrd();
maxOrd = globalValues.getMaxOrd();
}
if (maxOrd == 0) {
@ -200,9 +198,8 @@ public class ParentConstantScoreQuery extends Query {
if (globalValues != null) {
DocIdSetIterator innerIterator = childrenDocIdSet.iterator();
if (innerIterator != null) {
Ordinals.Docs globalOrdinals = globalValues.ordinals();
ChildrenDocIdIterator childrenDocIdIterator = new ChildrenDocIdIterator(
innerIterator, parentOrds, globalOrdinals
innerIterator, parentOrds, globalValues
);
return ConstantScorer.create(childrenDocIdIterator, this, queryWeight);
}
@ -215,9 +212,9 @@ public class ParentConstantScoreQuery extends Query {
private final class ChildrenDocIdIterator extends FilteredDocIdSetIterator {
private final LongBitSet parentOrds;
private final Ordinals.Docs globalOrdinals;
private final BytesValues.WithOrdinals globalOrdinals;
ChildrenDocIdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, Ordinals.Docs globalOrdinals) {
ChildrenDocIdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, BytesValues.WithOrdinals globalOrdinals) {
super(innerIterator);
this.parentOrds = parentOrds;
this.globalOrdinals = globalOrdinals;
@ -226,7 +223,7 @@ public class ParentConstantScoreQuery extends Query {
@Override
protected boolean match(int docId) {
int globalOrd = (int) globalOrdinals.getOrd(docId);
if (globalOrd != Ordinals.MISSING_ORDINAL) {
if (globalOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
return parentOrds.get(globalOrd);
} else {
return false;
@ -240,7 +237,7 @@ public class ParentConstantScoreQuery extends Query {
private final LongBitSet parentOrds;
private final IndexFieldData.WithOrdinals globalIfd;
private Ordinals.Docs globalOrdinals;
private BytesValues.WithOrdinals globalOrdinals;
ParentOrdsCollector(IndexFieldData.WithOrdinals globalIfd, long maxOrd) {
this.parentOrds = new LongBitSet(maxOrd);
@ -251,7 +248,7 @@ public class ParentConstantScoreQuery extends Query {
// It can happen that for particular segment no document exist for an specific type. This prevents NPE
if (globalOrdinals != null) {
long globalOrd = globalOrdinals.getOrd(doc);
if (globalOrd != Ordinals.MISSING_ORDINAL) {
if (globalOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
parentOrds.set(globalOrd);
}
}
@ -259,10 +256,7 @@ public class ParentConstantScoreQuery extends Query {
@Override
public void setNextReader(AtomicReaderContext readerContext) throws IOException {
BytesValues.WithOrdinals values = globalIfd.load(readerContext).getBytesValues();
if (values != null) {
globalOrdinals = values.ordinals();
}
globalOrdinals = globalIfd.load(readerContext).getBytesValues();
}
public long parentCount() {

View File

@ -36,7 +36,6 @@ import org.elasticsearch.common.util.FloatArray;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.internal.SearchContext.Lifetime;
@ -170,7 +169,6 @@ public class ParentQuery extends Query {
private Scorer scorer;
private BytesValues.WithOrdinals values;
private Ordinals.Docs globalOrdinals;
ParentOrdAndScoreCollector(SearchContext searchContext, IndexFieldData.WithOrdinals globalIfd) {
this.bigArrays = searchContext.bigArrays();
@ -182,9 +180,9 @@ public class ParentQuery extends Query {
@Override
public void collect(int doc) throws IOException {
// It can happen that for particular segment no document exist for an specific type. This prevents NPE
if (globalOrdinals != null) {
long globalOrdinal = globalOrdinals.getOrd(doc);
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
if (values != null) {
long globalOrdinal = values.getOrd(doc);
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
long parentIdx = parentIdxs.add(globalOrdinal);
if (parentIdx >= 0) {
scores = bigArrays.grow(scores, parentIdx + 1);
@ -204,9 +202,6 @@ public class ParentQuery extends Query {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
values = globalIfd.load(context).getBytesValues();
if (values != null) {
globalOrdinals = values.ordinals();
}
}
@Override
@ -268,8 +263,7 @@ public class ParentQuery extends Query {
return null;
}
Ordinals.Docs ordinals = bytesValues.ordinals();
return new ChildScorer(this, parentIdxs, scores, childrenDocSet.iterator(), ordinals);
return new ChildScorer(this, parentIdxs, scores, childrenDocSet.iterator(), bytesValues);
}
}
@ -279,12 +273,12 @@ public class ParentQuery extends Query {
private final LongHash parentIdxs;
private final FloatArray scores;
private final DocIdSetIterator childrenIterator;
private final Ordinals.Docs ordinals;
private final BytesValues.WithOrdinals ordinals;
private int currentChildDoc = -1;
private float currentScore;
ChildScorer(Weight weight, LongHash parentIdxs, FloatArray scores, DocIdSetIterator childrenIterator, Ordinals.Docs ordinals) {
ChildScorer(Weight weight, LongHash parentIdxs, FloatArray scores, DocIdSetIterator childrenIterator, BytesValues.WithOrdinals ordinals) {
super(weight);
this.parentIdxs = parentIdxs;
this.scores = scores;
@ -318,7 +312,7 @@ public class ParentQuery extends Query {
}
int globalOrdinal = (int) ordinals.getOrd(currentChildDoc);
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
continue;
}
@ -338,7 +332,7 @@ public class ParentQuery extends Query {
}
int globalOrdinal = (int) ordinals.getOrd(currentChildDoc);
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
return nextDoc();
}

View File

@ -22,7 +22,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator;
@ -61,14 +61,14 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
@Override
public SignificantStringTerms buildAggregation(long owningBucketOrdinal) {
assert owningBucketOrdinal == 0;
if (globalOrdinals == null) { // no context in this reader
if (globalValues == null) { // no context in this reader
return buildEmptyAggregation();
}
final int size;
if (bucketCountThresholds.getMinDocCount() == 0) {
// if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
size = (int) Math.min(globalOrdinals.getMaxOrd(), bucketCountThresholds.getShardSize());
size = (int) Math.min(globalValues.getMaxOrd(), bucketCountThresholds.getShardSize());
} else {
size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
}
@ -77,7 +77,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size);
SignificantStringTerms.Bucket spare = null;
for (long globalTermOrd = Ordinals.MIN_ORDINAL; globalTermOrd < globalOrdinals.getMaxOrd(); ++globalTermOrd) {
for (long globalTermOrd = BytesValues.WithOrdinals.MIN_ORDINAL; globalTermOrd < globalValues.getMaxOrd(); ++globalTermOrd) {
if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
continue;
}
@ -143,9 +143,9 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
@Override
public void collect(int doc, long owningBucketOrdinal) throws IOException {
numCollectedDocs++;
final int numOrds = globalOrdinals.setDocument(doc);
final int numOrds = globalValues.setDocument(doc);
for (int i = 0; i < numOrds; i++) {
final long globalOrd = globalOrdinals.nextOrd();
final long globalOrd = globalValues.nextOrd();
long bucketOrd = bucketOrds.add(globalOrd);
if (bucketOrd < 0) {
bucketOrd = -1 - bucketOrd;

View File

@ -30,8 +30,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.util.IntArray;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.GlobalOrdinalMapping;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.InternalAggregation;
@ -54,13 +53,12 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
protected final IncludeExclude includeExclude;
protected BytesValues.WithOrdinals globalValues;
protected Ordinals.Docs globalOrdinals;
// TODO: cache the acceptedGlobalOrdinals per aggregation definition.
// TODO: cache the acceptedglobalValues per aggregation definition.
// We can't cache this yet in ValuesSource, since ValuesSource is reused per field for aggs during the execution.
// If aggs with same field, but different include/exclude are defined, then the last defined one will override the
// first defined one.
// So currently for each instance of this aggregator the acceptedGlobalOrdinals will be computed, this is unnecessary
// So currently for each instance of this aggregator the acceptedglobalValues will be computed, this is unnecessary
// especially if this agg is on a second layer or deeper.
protected LongBitSet acceptedGlobalOrdinals;
@ -84,20 +82,19 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
@Override
public void setNextReader(AtomicReaderContext reader) {
globalValues = valuesSource.globalBytesValues();
globalOrdinals = globalValues.ordinals();
if (acceptedGlobalOrdinals != null) {
globalOrdinals = new FilteredOrdinals(globalOrdinals, acceptedGlobalOrdinals);
globalValues = new FilteredOrdinals(globalValues, acceptedGlobalOrdinals);
} else if (includeExclude != null) {
acceptedGlobalOrdinals = includeExclude.acceptedGlobalOrdinals(globalOrdinals, valuesSource);
globalOrdinals = new FilteredOrdinals(globalOrdinals, acceptedGlobalOrdinals);
acceptedGlobalOrdinals = includeExclude.acceptedGlobalOrdinals(globalValues, valuesSource);
globalValues = new FilteredOrdinals(globalValues, acceptedGlobalOrdinals);
}
}
@Override
public void collect(int doc, long owningBucketOrdinal) throws IOException {
final int numOrds = globalOrdinals.setDocument(doc);
final int numOrds = globalValues.setDocument(doc);
for (int i = 0; i < numOrds; i++) {
final long globalOrd = globalOrdinals.nextOrd();
final long globalOrd = globalValues.nextOrd();
collectExistingBucket(doc, globalOrd);
}
}
@ -113,20 +110,20 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
@Override
public InternalAggregation buildAggregation(long owningBucketOrdinal) {
if (globalOrdinals == null) { // no context in this reader
if (globalValues == null) { // no context in this reader
return buildEmptyAggregation();
}
final int size;
if (bucketCountThresholds.getMinDocCount() == 0) {
// if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
size = (int) Math.min(globalOrdinals.getMaxOrd(), bucketCountThresholds.getShardSize());
size = (int) Math.min(globalValues.getMaxOrd(), bucketCountThresholds.getShardSize());
} else {
size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
}
BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(this));
OrdBucket spare = new OrdBucket(-1, 0, null);
for (long globalTermOrd = Ordinals.MIN_ORDINAL; globalTermOrd < globalOrdinals.getMaxOrd(); ++globalTermOrd) {
for (long globalTermOrd = BytesValues.WithOrdinals.MIN_ORDINAL; globalTermOrd < globalValues.getMaxOrd(); ++globalTermOrd) {
if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
continue;
}
@ -228,9 +225,9 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
@Override
public void collect(int doc, long owningBucketOrdinal) throws IOException {
final int numOrds = globalOrdinals.setDocument(doc);
final int numOrds = globalValues.setDocument(doc);
for (int i = 0; i < numOrds; i++) {
final long globalOrd = globalOrdinals.nextOrd();
final long globalOrd = globalValues.nextOrd();
long bucketOrd = bucketOrds.add(globalOrd);
if (bucketOrd < 0) {
bucketOrd = -1 - bucketOrd;
@ -262,7 +259,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
private final IntArray segmentDocCounts;
private Ordinals.Docs segmentOrdinals;
private BytesValues.WithOrdinals segmentOrdinals;
private IntArray current;
public LowCardinality(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, long estimatedBucketCount,
@ -282,16 +279,14 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
@Override
public void setNextReader(AtomicReaderContext reader) {
if (segmentOrdinals != null && segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
if (segmentOrdinals != null && segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
mapSegmentCountsToGlobalCounts();
}
globalValues = valuesSource.globalBytesValues();
globalOrdinals = globalValues.ordinals();
BytesValues.WithOrdinals bytesValues = valuesSource.bytesValues();
segmentOrdinals = bytesValues.ordinals();
if (segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
segmentOrdinals = valuesSource.bytesValues();
if (segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
current = segmentDocCounts;
} else {
current = getDocCounts();
@ -300,7 +295,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
@Override
protected void doPostCollection() {
if (segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
if (segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
mapSegmentCountsToGlobalCounts();
}
}
@ -313,7 +308,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
private void mapSegmentCountsToGlobalCounts() {
// There is no public method in Ordinals.Docs that allows for this mapping...
// This is the cleanest way I can think of so far
GlobalOrdinalMapping mapping = (GlobalOrdinalMapping) globalOrdinals;
GlobalOrdinalMapping mapping = (GlobalOrdinalMapping) globalValues;
for (int i = 0; i < segmentDocCounts.size(); i++) {
final int inc = segmentDocCounts.set(i, 0);
if (inc == 0) {
@ -329,16 +324,17 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
}
}
private static final class FilteredOrdinals implements Ordinals.Docs {
private static final class FilteredOrdinals extends BytesValues.WithOrdinals {
private final Ordinals.Docs inner;
private final BytesValues.WithOrdinals inner;
private final LongBitSet accepted;
private long currentOrd;
private long[] buffer = new long[0];
private int bufferSlot;
private FilteredOrdinals(Ordinals.Docs inner, LongBitSet accepted) {
private FilteredOrdinals(BytesValues.WithOrdinals inner, LongBitSet accepted) {
super(inner.isMultiValued());
this.inner = inner;
this.accepted = accepted;
}
@ -348,18 +344,13 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
return inner.getMaxOrd();
}
@Override
public boolean isMultiValued() {
return inner.isMultiValued();
}
@Override
public long getOrd(int docId) {
long ord = inner.getOrd(docId);
if (accepted.get(ord)) {
return currentOrd = ord;
} else {
return currentOrd = Ordinals.MISSING_ORDINAL;
return currentOrd = MISSING_ORDINAL;
}
}
@ -389,5 +380,15 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
public long currentOrd() {
return currentOrd;
}
@Override
public BytesRef getValueByOrd(long ord) {
return inner.getValueByOrd(ord);
}
@Override
public BytesRef copyShared() {
return inner.copyShared();
}
}
}

View File

@ -28,7 +28,6 @@ import org.elasticsearch.common.collect.Iterators2;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.BytesRefHash;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.InternalAggregation;
@ -92,15 +91,14 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
/** Returns an iterator over the field data terms. */
private static Iterator<BytesRef> terms(final BytesValues.WithOrdinals bytesValues, boolean reverse) {
final Ordinals.Docs ordinals = bytesValues.ordinals();
if (reverse) {
return new UnmodifiableIterator<BytesRef>() {
long i = ordinals.getMaxOrd() - 1;
long i = bytesValues.getMaxOrd() - 1;
@Override
public boolean hasNext() {
return i >= Ordinals.MIN_ORDINAL;
return i >= BytesValues.WithOrdinals.MIN_ORDINAL;
}
@Override
@ -113,11 +111,11 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
} else {
return new UnmodifiableIterator<BytesRef>() {
long i = Ordinals.MIN_ORDINAL;
long i = BytesValues.WithOrdinals.MIN_ORDINAL;
@Override
public boolean hasNext() {
return i < ordinals.getMaxOrd();
return i < bytesValues.getMaxOrd();
}
@Override

View File

@ -19,11 +19,14 @@
package org.elasticsearch.search.aggregations.bucket.terms.support;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.internal.SearchContext;
@ -75,7 +78,7 @@ public class IncludeExclude {
/**
* Computes which global ordinals are accepted by this IncludeExclude instance.
*/
public LongBitSet acceptedGlobalOrdinals(Ordinals.Docs globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) {
public LongBitSet acceptedGlobalOrdinals(BytesValues.WithOrdinals globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) {
TermsEnum globalTermsEnum = valueSource.getGlobalTermsEnum();
LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getMaxOrd());
try {

View File

@ -34,7 +34,6 @@ import org.elasticsearch.common.util.ObjectArray;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.LongValues;
import org.elasticsearch.index.fielddata.MurmurHash3Values;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
@ -92,7 +91,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
final BytesValues bytesValues = valuesSource.bytesValues();
if (bytesValues instanceof BytesValues.WithOrdinals) {
BytesValues.WithOrdinals values = (BytesValues.WithOrdinals) bytesValues;
final long maxOrd = values.ordinals().getMaxOrd();
final long maxOrd = values.getMaxOrd();
if (maxOrd == 0) {
return new EmptyCollector();
}
@ -231,15 +230,13 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
private final BigArrays bigArrays;
private final BytesValues.WithOrdinals values;
private final Ordinals.Docs ordinals;
private final int maxOrd;
private final HyperLogLogPlusPlus counts;
private ObjectArray<FixedBitSet> visitedOrds;
OrdinalsCollector(HyperLogLogPlusPlus counts, BytesValues.WithOrdinals values, BigArrays bigArrays) {
ordinals = values.ordinals();
Preconditions.checkArgument(ordinals.getMaxOrd() <= Integer.MAX_VALUE);
maxOrd = (int) ordinals.getMaxOrd();
Preconditions.checkArgument(values.getMaxOrd() <= Integer.MAX_VALUE);
maxOrd = (int) values.getMaxOrd();
this.bigArrays = bigArrays;
this.counts = counts;
this.values = values;
@ -254,9 +251,9 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
bits = new FixedBitSet(maxOrd);
visitedOrds.set(bucketOrd, bits);
}
final int valueCount = ordinals.setDocument(doc);
final int valueCount = values.setDocument(doc);
for (int i = 0; i < valueCount; ++i) {
bits.set((int) ordinals.nextOrd());
bits.set((int) values.nextOrd());
}
}

View File

@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.TopReaderContextAware;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.AtomicFieldData.Order;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.SortedAndUnique.SortedUniqueBytesValues;
import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues;
@ -91,8 +90,25 @@ public abstract class ValuesSource {
metaData.uniqueness = Uniqueness.UNIQUE;
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
AtomicFieldData fieldData = indexFieldData.load(readerContext);
metaData.multiValued |= fieldData.isMultiValued();
metaData.maxAtomicUniqueValuesCount = Math.max(metaData.maxAtomicUniqueValuesCount, fieldData.getNumberUniqueValues());
if (fieldData instanceof AtomicFieldData.WithOrdinals<?>) {
AtomicFieldData.WithOrdinals<?> fd = (AtomicFieldData.WithOrdinals<?>) fieldData;
BytesValues.WithOrdinals values = fd.getBytesValues();
metaData.multiValued |= values.isMultiValued();
metaData.maxAtomicUniqueValuesCount = Math.max(metaData.maxAtomicUniqueValuesCount, values.getMaxOrd());
} else if (fieldData instanceof AtomicNumericFieldData) {
AtomicNumericFieldData fd = (AtomicNumericFieldData) fieldData;
DoubleValues values = fd.getDoubleValues();
metaData.multiValued |= values.isMultiValued();
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
} else if (fieldData instanceof AtomicGeoPointFieldData<?>) {
AtomicGeoPointFieldData<?> fd = (AtomicGeoPointFieldData<?>) fieldData;
GeoPointValues values = fd.getGeoPointValues();
metaData.multiValued |= values.isMultiValued();
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
} else {
metaData.multiValued = true;
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
}
}
return metaData;
}
@ -243,8 +259,7 @@ public abstract class ValuesSource {
IndexFieldData.WithOrdinals<?> globalFieldData = indexFieldData.loadGlobal(indexReader);
AtomicFieldData.WithOrdinals afd = globalFieldData.load(atomicReaderContext);
BytesValues.WithOrdinals values = afd.getBytesValues();
Ordinals.Docs ordinals = values.ordinals();
return maxOrd = ordinals.getMaxOrd();
return maxOrd = values.getMaxOrd();
}
}
@ -349,6 +364,7 @@ public abstract class ValuesSource {
}
static class SortedUniqueBytesValues extends BytesValues {
final BytesRef scratch = new BytesRef();
final BytesValues delegate;
int[] indices = new int[1]; // at least one
final BytesRefArray bytes;
@ -758,6 +774,7 @@ public abstract class ValuesSource {
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues {
private final BytesRef scratch = new BytesRef();
private final ValuesSource source;
private final SearchScript script;

View File

@ -33,6 +33,7 @@ import java.util.Iterator;
*/
public class ScriptBytesValues extends BytesValues implements ScriptValues {
private final BytesRef scratch = new BytesRef();
final SearchScript script;
private Iterator<?> iter;
@ -50,7 +51,6 @@ public class ScriptBytesValues extends BytesValues implements ScriptValues {
@Override
public int setDocument(int docId) {
this.docId = docId;
script.setNextDocId(docId);
value = script.run();

View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.collect.BoundedTreeSet;
import org.elasticsearch.common.recycler.Recycler;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.DoubleFacetAggregatorBase;
import org.elasticsearch.search.facet.FacetExecutor;
@ -74,19 +73,11 @@ public class TermsDoubleFacetExecutor extends FacetExecutor {
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
int maxDoc = readerContext.reader().maxDoc();
DoubleValues values = indexFieldData.load(readerContext).getDoubleValues();
if (values instanceof DoubleValues.WithOrdinals) {
DoubleValues.WithOrdinals valuesWithOrds = (DoubleValues.WithOrdinals) values;
Ordinals.Docs ordinals = valuesWithOrds.ordinals();
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
facets.v().putIfAbsent(valuesWithOrds.getValueByOrd(ord), 0);
}
} else {
for (int docId = 0; docId < maxDoc; docId++) {
int numValues = values.setDocument(docId);
DoubleIntOpenHashMap map = facets.v();
for (int i = 0; i < numValues; i++) {
map.putIfAbsent(values.nextValue(), 0);
}
for (int docId = 0; docId < maxDoc; docId++) {
int numValues = values.setDocument(docId);
DoubleIntOpenHashMap map = facets.v();
for (int i = 0; i < numValues; i++) {
map.putIfAbsent(values.nextValue(), 0);
}
}
}

View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.collect.BoundedTreeSet;
import org.elasticsearch.common.recycler.Recycler;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.LongValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.InternalFacet;
@ -73,19 +72,11 @@ public class TermsLongFacetExecutor extends FacetExecutor {
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
int maxDoc = readerContext.reader().maxDoc();
LongValues values = indexFieldData.load(readerContext).getLongValues();
if (values instanceof LongValues.WithOrdinals) {
LongValues.WithOrdinals valuesWithOrds = (LongValues.WithOrdinals) values;
Ordinals.Docs ordinals = valuesWithOrds.ordinals();
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
facets.v().putIfAbsent(valuesWithOrds.getValueByOrd(ord), 0);
}
} else {
for (int docId = 0; docId < maxDoc; docId++) {
final int numValues = values.setDocument(docId);
final LongIntOpenHashMap v = facets.v();
for (int i = 0; i < numValues; i++) {
v.putIfAbsent(values.nextValue(), 0);
}
for (int docId = 0; docId < maxDoc; docId++) {
final int numValues = values.setDocument(docId);
final LongIntOpenHashMap v = facets.v();
for (int i = 0; i < numValues; i++) {
v.putIfAbsent(values.nextValue(), 0);
}
}
}

View File

@ -25,7 +25,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.InternalFacet;
@ -131,9 +130,8 @@ public class TermsStringFacetExecutor extends FacetExecutor {
int maxDoc = readerContext.reader().maxDoc();
if (indexFieldData instanceof IndexFieldData.WithOrdinals) {
BytesValues.WithOrdinals values = ((IndexFieldData.WithOrdinals) indexFieldData).load(readerContext).getBytesValues();
Ordinals.Docs ordinals = values.ordinals();
// 0 = docs with no value for field, so start from 1 instead
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
for (long ord = BytesValues.WithOrdinals.MIN_ORDINAL; ord < values.getMaxOrd(); ord++) {
BytesRef value = values.getValueByOrd(ord);
aggregator.addValue(value, value.hashCode(), values);
}

View File

@ -33,7 +33,6 @@ import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.IntArray;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.InternalFacet;
import org.elasticsearch.search.facet.terms.TermsFacet;
@ -200,14 +199,13 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
private long total;
private BytesValues.WithOrdinals values;
private ReaderAggregator current;
private Ordinals.Docs ordinals;
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
if (current != null) {
missing += current.missing;
total += current.total;
if (current.values.ordinals().getMaxOrd() > Ordinals.MIN_ORDINAL) {
if (current.values.getMaxOrd() > BytesValues.WithOrdinals.MIN_ORDINAL) {
aggregators.add(current);
} else {
Releasables.close(current);
@ -215,15 +213,14 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
}
values = indexFieldData.load(context).getBytesValues();
current = new ReaderAggregator(values, ordinalsCacheAbove, cacheRecycler);
ordinals = values.ordinals();
}
@Override
public void collect(int doc) throws IOException {
final int length = ordinals.setDocument(doc);
final int length = values.setDocument(doc);
int missing = 1;
for (int i = 0; i < length; i++) {
current.onOrdinal(doc, ordinals.nextOrd());
current.onOrdinal(doc, values.nextOrd());
missing = 0;
}
current.incrementMissing(missing);
@ -235,7 +232,7 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
missing += current.missing;
total += current.total;
// if we have values for this one, add it
if (current.values.ordinals().getMaxOrd() > Ordinals.MIN_ORDINAL) {
if (current.values.getMaxOrd() > BytesValues.WithOrdinals.MIN_ORDINAL) {
aggregators.add(current);
} else {
Releasables.close(current);
@ -254,14 +251,14 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
final BytesValues.WithOrdinals values;
final IntArray counts;
int missing = 0;
long position = Ordinals.MIN_ORDINAL - 1;
long position = BytesValues.WithOrdinals.MIN_ORDINAL - 1;
BytesRef current;
int total;
public ReaderAggregator(BytesValues.WithOrdinals values, int ordinalsCacheLimit, CacheRecycler cacheRecycler) {
this.values = values;
this.maxOrd = values.ordinals().getMaxOrd();
this.maxOrd = values.getMaxOrd();
this.counts = bigArrays.newIntArray(maxOrd);
}

View File

@ -34,15 +34,12 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.SizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.discovery.Discovery;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.transport.TransportModule;
import java.io.IOException;
import java.util.*;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@ -174,20 +171,14 @@ public class GlobalOrdinalsBenchmark {
System.out.println("--> Number of docs in index: " + COUNT);
List<StatsResult> stats = new ArrayList<>();
int[] thresholds = new int[]{2048};
for (int threshold : thresholds) {
updateThresholdInMapping(threshold);
System.out.println("--> Threshold: " + threshold);
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
String fieldName = "field_" + fieldSuffix;
String name = "global_ordinals-" + fieldName;
if (USE_DOC_VALUES) {
fieldName = fieldName + ".doc_values";
name = name + "_doc_values"; // can't have . in agg name
}
stats.add(terms(name, fieldName, "global_ordinals_low_cardinality"));
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
String fieldName = "field_" + fieldSuffix;
String name = "global_ordinals-" + fieldName;
if (USE_DOC_VALUES) {
fieldName = fieldName + ".doc_values";
name = name + "_doc_values"; // can't have . in agg name
}
stats.add(terms(name, fieldName, "global_ordinals_low_cardinality"));
}
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
@ -211,22 +202,6 @@ public class GlobalOrdinalsBenchmark {
node.close();
}
private static void updateThresholdInMapping(int threshold) throws IOException {
XContentBuilder builder = jsonBuilder().startObject().startObject(TYPE_NAME).startObject("properties");
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
builder.startObject("field_" + fieldSuffix)
.field("type", "string")
.field("index", "not_analyzed")
.startObject("fielddata")
.field(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_KEY, threshold)
.endObject()
.endObject();
}
builder = builder.endObject().endObject().endObject();
client.admin().indices().preparePutMapping(INDEX_NAME).setType(TYPE_NAME).setSource(builder).get();
}
private static StatsResult terms(String name, String field, String executionHint) {
long totalQueryTime;// LM VALUE

View File

@ -43,7 +43,6 @@ import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Test;
@ -427,7 +426,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
public void testGlobalOrdinals() throws Exception {
fillExtendedMvSet();
refreshReader();
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_ordinals", "fixed"));
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_values", "fixed"));
IndexFieldData.WithOrdinals ifd = getForField(fieldDataType, "value");
IndexFieldData.WithOrdinals globalOrdinals = ifd.loadGlobal(topLevelReader);
assertThat(topLevelReader.leaves().size(), equalTo(3));
@ -436,68 +435,65 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class));
AtomicFieldData.WithOrdinals afd = globalOrdinals.load(topLevelReader.leaves().get(0));
BytesValues.WithOrdinals values = afd.getBytesValues();
Ordinals.Docs ordinals = afd.getBytesValues().ordinals();
assertThat(ordinals.setDocument(0), equalTo(2));
long ord = ordinals.nextOrd();
assertThat(values.setDocument(0), equalTo(2));
long ord = values.nextOrd();
assertThat(ord, equalTo(3l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("02"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(5l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("04"));
assertThat(ordinals.setDocument(1), equalTo(0));
assertThat(ordinals.setDocument(2), equalTo(1));
ord = ordinals.nextOrd();
assertThat(values.setDocument(1), equalTo(0));
assertThat(values.setDocument(2), equalTo(1));
ord = values.nextOrd();
assertThat(ord, equalTo(4l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("03"));
// Second segment
afd = globalOrdinals.load(topLevelReader.leaves().get(1));
values = afd.getBytesValues();
ordinals = afd.getBytesValues().ordinals();
assertThat(ordinals.setDocument(0), equalTo(3));
ord = ordinals.nextOrd();
assertThat(values.setDocument(0), equalTo(3));
ord = values.nextOrd();
assertThat(ord, equalTo(5l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("04"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(6l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("05"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(7l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("06"));
assertThat(ordinals.setDocument(1), equalTo(3));
ord = ordinals.nextOrd();
assertThat(values.setDocument(1), equalTo(3));
ord = values.nextOrd();
assertThat(ord, equalTo(7l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("06"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(8l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("07"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(9l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("08"));
assertThat(ordinals.setDocument(2), equalTo(0));
assertThat(ordinals.setDocument(3), equalTo(3));
ord = ordinals.nextOrd();
assertThat(values.setDocument(2), equalTo(0));
assertThat(values.setDocument(3), equalTo(3));
ord = values.nextOrd();
assertThat(ord, equalTo(9l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("08"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(10l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("09"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(11l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("10"));
// Third segment
afd = globalOrdinals.load(topLevelReader.leaves().get(2));
values = afd.getBytesValues();
ordinals = afd.getBytesValues().ordinals();
assertThat(ordinals.setDocument(0), equalTo(3));
ord = ordinals.nextOrd();
assertThat(values.setDocument(0), equalTo(3));
ord = values.nextOrd();
assertThat(ord, equalTo(0l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!08"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(1l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!09"));
ord = ordinals.nextOrd();
ord = values.nextOrd();
assertThat(ord, equalTo(2l));
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!10"));
}
@ -542,7 +538,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception {
fillExtendedMvSet();
refreshReader();
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_ordinals", "fixed"));
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_values", "fixed"));
IndexFieldData.WithOrdinals ifd = getForField(fieldDataType, "value");
IndexFieldData.WithOrdinals globalOrdinals = ifd.loadGlobal(topLevelReader);
assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));

View File

@ -38,8 +38,6 @@ import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.unit.DistanceUnit.Distance;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperTestUtils;
import org.elasticsearch.index.mapper.ParsedDocument;
@ -404,10 +402,8 @@ public class DuelFieldDataTests extends AbstractFieldDataTests {
BytesValues.WithOrdinals left = fieldData.load(readerContext).getBytesValues();
fieldData.clear();
BytesValues.WithOrdinals right = fieldData.loadGlobal(topLevelReader).load(topLevelReader.leaves().get(0)).getBytesValues();
Docs leftOrds = left.ordinals();
Docs rightOrds = right.ordinals();
assertEquals(leftOrds.getMaxOrd(), rightOrds.getMaxOrd());
for (long ord = Ordinals.MIN_ORDINAL; ord < leftOrds.getMaxOrd(); ++ord) {
assertEquals(left.getMaxOrd(), right.getMaxOrd());
for (long ord = BytesValues.WithOrdinals.MIN_ORDINAL; ord < left.getMaxOrd(); ++ord) {
assertEquals(left.getValueByOrd(ord), right.getValueByOrd(ord));
}
}

View File

@ -25,7 +25,6 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.AtomicFieldData.WithOrdinals;
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.junit.Test;
import java.util.Random;
@ -72,8 +71,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(2L, equalTo(ordinals.getMaxOrd()));
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
}
@ -84,8 +82,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(1L, equalTo(ordinals.getMaxOrd()));
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("5"));
}
@ -96,8 +93,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "med_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(2L, equalTo(ordinals.getMaxOrd()));
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
}
@ -109,8 +105,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "med_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(2L, equalTo(ordinals.getMaxOrd()));
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
}
@ -125,8 +120,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(1L, equalTo(ordinals.getMaxOrd()));
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("100"));
}
}
@ -169,8 +163,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(1L, equalTo(ordinals.getMaxOrd()));
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("5"));
}
{
@ -180,8 +173,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
Docs ordinals = bytesValues.ordinals();
assertThat(2L, equalTo(ordinals.getMaxOrd()));
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("5"));
}

View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
@ -103,7 +104,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
}
});
Ordinals ords = creationMultiOrdinals(builder);
Ordinals.Docs docs = ords.ordinals();
BytesValues.WithOrdinals docs = ords.ordinals();
int docId = ordsAndIds.get(0).id;
List<Long> docOrds = new ArrayList<>();
for (OrdAndId ordAndId : ordsAndIds) {
@ -125,7 +126,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
assertIter(docs, docId, array);
}
for (int i = docId + 1; i < ordAndId.id; i++) {
assertThat(docs.getOrd(i), equalTo(Ordinals.MISSING_ORDINAL));
assertThat(docs.getOrd(i), equalTo(BytesValues.WithOrdinals.MISSING_ORDINAL));
}
docId = ordAndId.id;
docOrds.clear();
@ -209,11 +210,11 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
};
Ordinals ordinals = creationMultiOrdinals(builder);
Ordinals.Docs docs = ordinals.ordinals();
BytesValues.WithOrdinals docs = ordinals.ordinals();
assertEquals(docs, ordinalPlan);
}
protected static void assertIter(Ordinals.Docs docs, int docId, long... expectedOrdinals) {
protected static void assertIter(BytesValues.WithOrdinals docs, int docId, long... expectedOrdinals) {
assertThat(docs.setDocument(docId), equalTo(expectedOrdinals.length));
for (long expectedOrdinal : expectedOrdinals) {
assertThat(docs.nextOrd(), equalTo(expectedOrdinal));
@ -261,11 +262,11 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
};
Ordinals ordinals = new MultiOrdinals(builder, PackedInts.FASTEST);
Ordinals.Docs docs = ordinals.ordinals();
BytesValues.WithOrdinals docs = ordinals.ordinals();
assertEquals(docs, ordinalPlan);
}
private void assertEquals(Ordinals.Docs docs, long[][] ordinalPlan) {
private void assertEquals(BytesValues.WithOrdinals docs, long[][] ordinalPlan) {
long maxOrd = 0;
for (int doc = 0; doc < ordinalPlan.length; ++doc) {
if (ordinalPlan[doc].length > 0) {

View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
@ -26,7 +27,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
/**
@ -50,7 +50,7 @@ public class SingleOrdinalsTests extends ElasticsearchTestCase {
Ordinals ords = builder.build(ImmutableSettings.EMPTY);
assertThat(ords, instanceOf(SinglePackedOrdinals.class));
Ordinals.Docs docs = ords.ordinals();
BytesValues.WithOrdinals docs = ords.ordinals();
for (Map.Entry<Integer, Long> entry : controlDocToOrdinal.entrySet()) {
assertThat(entry.getValue(), equalTo(docs.getOrd(entry.getKey())));

View File

@ -26,9 +26,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.RangeFilterBuilder;
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
@ -155,7 +153,6 @@ public class RandomTests extends ElasticsearchIntegrationTest {
final IntOpenHashSet valuesSet = new IntOpenHashSet();
cluster().wipeIndices("idx");
prepareCreate("idx")
.setSettings(ImmutableSettings.builder().put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, randomIntBetween(1, maxNumTerms)))
.addMapping("type", jsonBuilder().startObject()
.startObject("type")
.startObject("properties")

View File

@ -37,6 +37,7 @@ public class FieldDataSourceTests extends ElasticsearchTestCase {
final boolean multiValued = randomBoolean();
final int maxLength = rarely() ? 3 : 10;
return new BytesValues(multiValued) {
private final BytesRef scratch = new BytesRef();
BytesRef previous;
@Override
public int setDocument(int docId) {

View File

@ -63,7 +63,6 @@ import org.elasticsearch.index.cache.filter.FilterCacheModule;
import org.elasticsearch.index.cache.filter.none.NoneFilterCache;
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.plugins.PluginsService;
@ -317,7 +316,6 @@ public final class InternalTestCluster extends TestCluster {
}
}
builder.put("plugins.isolation", random.nextBoolean());
builder.put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, 1 + random.nextInt(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_DEFAULT));
if (random.nextInt(10) == 0) {
builder.put(EsExecutors.PROCESSORS, 1 + random.nextInt(AbstractRandomizedTest.TESTS_PROCESSORS));
} else {