Fielddata: Merge ordinals APIs into BytesValues.WithOrdinals.
Mid-term we should switch from `BytesValues` to Lucene's doc values APIs, in particular the `SortedSetDocValues` class. While `BytesValues.WithOrdinals` and SortedSetDocValues expose the same functionality, `BytesValues.WithOrdinals` exposes its ordinals via a different `Ordinals.Docs` object while `SortedSetDocValues` exposes them on the same object as the one that holds the values. This commit merges ordinals into `BytesValues.WithOrdinals` in order to make both classes even closer. Global ordinals were a bit tricky to migrate so I just changed them to use Lucene's OrdinalMap that will soon (LUCENE-5767, scheduled for 4.9) have the same optimizations as our global ordinals. Close #6524
This commit is contained in:
parent
703dbff83d
commit
9b02b5061b
|
@ -45,10 +45,10 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
|
|||
if (isFloat) {
|
||||
final DoubleValues values = getDoubleValues();
|
||||
return new BytesValues(values.isMultiValued()) {
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
return values.setDocument(docId);
|
||||
}
|
||||
|
||||
|
@ -67,10 +67,10 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
|
|||
} else {
|
||||
final LongValues values = getLongValues();
|
||||
return new BytesValues(values.isMultiValued()) {
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
return values.setDocument(docId);
|
||||
}
|
||||
|
||||
|
|
|
@ -20,29 +20,20 @@
|
|||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
|
||||
import org.elasticsearch.index.fielddata.plain.AtomicFieldDataWithOrdinalsTermsEnum;
|
||||
|
||||
/**
|
||||
* The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
|
||||
*/
|
||||
public interface AtomicFieldData<Script extends ScriptDocValues> extends RamUsage {
|
||||
|
||||
/**
|
||||
* If this method returns false, this means that no document has multiple values. However this method may return true even if all
|
||||
* documents are single-valued. So this method is useful for performing optimizations when the single-value case makes the problem
|
||||
* simpler but cannot be used to actually check whether this instance is multi-valued.
|
||||
*/
|
||||
boolean isMultiValued();
|
||||
|
||||
/**
|
||||
* An upper limit of the number of unique values in this atomic field data.
|
||||
*/
|
||||
long getNumberUniqueValues();
|
||||
|
||||
/**
|
||||
* Use a non thread safe (lightweight) view of the values as bytes.
|
||||
*/
|
||||
BytesValues getBytesValues();
|
||||
|
||||
|
||||
/**
|
||||
* Returns a "scripting" based values.
|
||||
*/
|
||||
|
@ -55,6 +46,65 @@ public interface AtomicFieldData<Script extends ScriptDocValues> extends RamUsag
|
|||
|
||||
interface WithOrdinals<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
|
||||
|
||||
public static final WithOrdinals<ScriptDocValues.Strings> EMPTY = new WithOrdinals<ScriptDocValues.Strings>() {
|
||||
|
||||
@Override
|
||||
public Strings getScriptValues() {
|
||||
return new ScriptDocValues.Strings(getBytesValues());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new BytesValues.WithOrdinals(false) {
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
return MISSING_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
return MISSING_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long currentOrd() {
|
||||
return MISSING_ORDINAL;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public TermsEnum getTermsEnum() {
|
||||
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Use a non thread safe (lightweight) view of the values as bytes.
|
||||
* @param needsHashes
|
||||
|
|
|
@ -32,10 +32,10 @@ public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> im
|
|||
public BytesValues getBytesValues() {
|
||||
final GeoPointValues values = getGeoPointValues();
|
||||
return new BytesValues(values.isMultiValued()) {
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
return values.setDocument(docId);
|
||||
}
|
||||
|
||||
|
|
|
@ -19,10 +19,9 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.apache.lucene.index.SortedSetDocValues;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
|
||||
/**
|
||||
* A state-full lightweight per document set of <code>byte[]</code> values.
|
||||
|
@ -44,11 +43,7 @@ public abstract class BytesValues {
|
|||
*/
|
||||
public static final BytesValues EMPTY = new Empty();
|
||||
|
||||
private boolean multiValued;
|
||||
|
||||
protected final BytesRef scratch = new BytesRef();
|
||||
|
||||
protected int docId = -1;
|
||||
private final boolean multiValued;
|
||||
|
||||
/**
|
||||
* Creates a new {@link BytesValues} instance
|
||||
|
@ -114,20 +109,41 @@ public abstract class BytesValues {
|
|||
*/
|
||||
public static abstract class WithOrdinals extends BytesValues {
|
||||
|
||||
protected final Docs ordinals;
|
||||
public static final long MIN_ORDINAL = 0;
|
||||
public static final long MISSING_ORDINAL = SortedSetDocValues.NO_MORE_ORDS;
|
||||
|
||||
protected WithOrdinals(Ordinals.Docs ordinals) {
|
||||
super(ordinals.isMultiValued());
|
||||
this.ordinals = ordinals;
|
||||
protected WithOrdinals(boolean multiValued) {
|
||||
super(multiValued);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the associated ordinals instance.
|
||||
* @return the associated ordinals instance.
|
||||
* Returns total unique ord count;
|
||||
*/
|
||||
public Ordinals.Docs ordinals() {
|
||||
return ordinals;
|
||||
}
|
||||
public abstract long getMaxOrd();
|
||||
|
||||
/**
|
||||
* The ordinal that maps to the relevant docId. If it has no value, returns
|
||||
* <tt>0</tt>.
|
||||
*/
|
||||
public abstract long getOrd(int docId);
|
||||
|
||||
/**
|
||||
* Returns the next ordinal for the current docID set to {@link #setDocument(int)}.
|
||||
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
|
||||
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
|
||||
* is undefined.
|
||||
*
|
||||
* Note: This method will never return <tt>0</tt>.
|
||||
*
|
||||
* @return the next ordinal for the current docID set to {@link #setDocument(int)}.
|
||||
*/
|
||||
public abstract long nextOrd();
|
||||
|
||||
/**
|
||||
* Returns the current ordinal in the iteration
|
||||
* @return the current ordinal in the iteration
|
||||
*/
|
||||
public abstract long currentOrd();
|
||||
|
||||
/**
|
||||
* Returns the value for the given ordinal.
|
||||
|
@ -137,18 +153,9 @@ public abstract class BytesValues {
|
|||
*/
|
||||
public abstract BytesRef getValueByOrd(long ord);
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
int length = ordinals.setDocument(docId);
|
||||
assert (ordinals.getOrd(docId) != Ordinals.MISSING_ORDINAL) == length > 0 : "Doc: [" + docId + "] hasValue: [" + (ordinals.getOrd(docId) != Ordinals.MISSING_ORDINAL) + "] but length is [" + length + "]";
|
||||
return length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef nextValue() {
|
||||
assert docId != -1;
|
||||
return getValueByOrd(ordinals.nextOrd());
|
||||
return getValueByOrd(nextOrd());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -20,8 +20,6 @@
|
|||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
|
||||
/**
|
||||
* A state-full lightweight per document set of <code>double</code> values.
|
||||
|
@ -100,21 +98,13 @@ public abstract class DoubleValues {
|
|||
*/
|
||||
public static abstract class WithOrdinals extends DoubleValues {
|
||||
|
||||
protected final Docs ordinals;
|
||||
protected final BytesValues.WithOrdinals ordinals;
|
||||
|
||||
protected WithOrdinals(Ordinals.Docs ordinals) {
|
||||
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals.isMultiValued());
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the associated ordinals instance.
|
||||
* @return the associated ordinals instance.
|
||||
*/
|
||||
public Docs ordinals() {
|
||||
return ordinals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the value for the given ordinal.
|
||||
* @param ord the ordinal to lookup.
|
||||
|
|
|
@ -20,8 +20,6 @@
|
|||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
|
||||
/**
|
||||
* A state-full lightweight per document set of <code>long</code> values.
|
||||
|
@ -101,21 +99,13 @@ public abstract class LongValues {
|
|||
*/
|
||||
public static abstract class WithOrdinals extends LongValues {
|
||||
|
||||
protected final Docs ordinals;
|
||||
protected final BytesValues.WithOrdinals ordinals;
|
||||
|
||||
protected WithOrdinals(Ordinals.Docs ordinals) {
|
||||
protected WithOrdinals(BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals.isMultiValued());
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the associated ordinals instance.
|
||||
* @return the associated ordinals instance.
|
||||
*/
|
||||
public Docs ordinals() {
|
||||
return this.ordinals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the value for the given ordinal.
|
||||
* @param ord the ordinal to lookup.
|
||||
|
|
|
@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldComparator;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.search.MultiValueMode;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -142,13 +141,11 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
}
|
||||
|
||||
class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
|
||||
final Ordinals.Docs readerOrds;
|
||||
final BytesValues.WithOrdinals termsIndex;
|
||||
|
||||
public PerSegmentComparator(BytesValues.WithOrdinals termsIndex) {
|
||||
this.readerOrds = termsIndex.ordinals();
|
||||
this.termsIndex = termsIndex;
|
||||
if (readerOrds.getMaxOrd() > Long.MAX_VALUE / 4) {
|
||||
if (termsIndex.getMaxOrd() > Long.MAX_VALUE / 4) {
|
||||
throw new IllegalStateException("Current terms index pretends it has more than " + (Long.MAX_VALUE / 4) + " ordinals, which is unsupported by this impl");
|
||||
}
|
||||
}
|
||||
|
@ -192,21 +189,21 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
}
|
||||
|
||||
protected long getOrd(int doc) {
|
||||
return readerOrds.getOrd(doc);
|
||||
return termsIndex.getOrd(doc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) {
|
||||
assert bottomSlot != -1;
|
||||
final long docOrd = getOrd(doc);
|
||||
final long comparableOrd = docOrd == Ordinals.MISSING_ORDINAL ? missingOrd : docOrd << 2;
|
||||
final long comparableOrd = docOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? missingOrd : docOrd << 2;
|
||||
return Long.compare(bottomOrd, comparableOrd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTop(int doc) throws IOException {
|
||||
final long ord = getOrd(doc);
|
||||
if (ord == Ordinals.MISSING_ORDINAL) {
|
||||
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
return compareTopMissing();
|
||||
} else {
|
||||
final long comparableOrd = ord << 2;
|
||||
|
@ -233,7 +230,7 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
final long ord = getOrd(doc);
|
||||
if (ord == Ordinals.MISSING_ORDINAL) {
|
||||
if (ord == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
ords[slot] = missingOrd;
|
||||
values[slot] = missingValue;
|
||||
} else {
|
||||
|
@ -259,13 +256,13 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
private boolean consistentInsertedOrd(BytesValues.WithOrdinals termsIndex, long ord, BytesRef value) {
|
||||
final long previousOrd = ord >> 2;
|
||||
final long nextOrd = previousOrd + 1;
|
||||
final BytesRef previous = previousOrd == Ordinals.MISSING_ORDINAL ? null : termsIndex.getValueByOrd(previousOrd);
|
||||
final BytesRef previous = previousOrd == BytesValues.WithOrdinals.MISSING_ORDINAL ? null : termsIndex.getValueByOrd(previousOrd);
|
||||
if ((ord & 3) == 0) { // there was an existing ord with the inserted value
|
||||
assert compareValues(previous, value) == 0;
|
||||
} else {
|
||||
assert compareValues(previous, value) < 0;
|
||||
}
|
||||
if (nextOrd < termsIndex.ordinals().getMaxOrd()) {
|
||||
if (nextOrd < termsIndex.getMaxOrd()) {
|
||||
final BytesRef next = termsIndex.getValueByOrd(nextOrd);
|
||||
assert compareValues(value, next) < 0;
|
||||
}
|
||||
|
@ -276,10 +273,10 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
private long ordInCurrentReader(BytesValues.WithOrdinals termsIndex, BytesRef value) {
|
||||
final long ord;
|
||||
if (value == null) {
|
||||
ord = Ordinals.MISSING_ORDINAL << 2;
|
||||
ord = BytesValues.WithOrdinals.MISSING_ORDINAL << 2;
|
||||
} else {
|
||||
final long docOrd = binarySearch(termsIndex, value);
|
||||
if (docOrd >= Ordinals.MIN_ORDINAL) {
|
||||
if (docOrd >= BytesValues.WithOrdinals.MIN_ORDINAL) {
|
||||
// value exists in the current segment
|
||||
ord = docOrd << 2;
|
||||
} else {
|
||||
|
@ -295,16 +292,14 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
@Override
|
||||
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
|
||||
termsIndex = indexFieldData.load(context).getBytesValues();
|
||||
assert termsIndex.ordinals() != null;
|
||||
missingOrd = ordInCurrentReader(termsIndex, missingValue);
|
||||
assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
|
||||
FieldComparator<BytesRef> perSegComp = null;
|
||||
assert termsIndex.ordinals() != null;
|
||||
if (termsIndex.isMultiValued()) {
|
||||
perSegComp = new PerSegmentComparator(termsIndex) {
|
||||
@Override
|
||||
protected long getOrd(int doc) {
|
||||
return getRelevantOrd(readerOrds, doc, sortMode);
|
||||
return getRelevantOrd(termsIndex, doc, sortMode);
|
||||
}
|
||||
};
|
||||
} else {
|
||||
|
@ -359,12 +354,12 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
}
|
||||
|
||||
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
|
||||
return binarySearch(a, key, Ordinals.MIN_ORDINAL, a.ordinals().getMaxOrd() - 1);
|
||||
return binarySearch(a, key, BytesValues.WithOrdinals.MIN_ORDINAL, a.getMaxOrd() - 1);
|
||||
}
|
||||
|
||||
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key, long low, long high) {
|
||||
assert low != Ordinals.MISSING_ORDINAL;
|
||||
assert high == Ordinals.MISSING_ORDINAL || (a.getValueByOrd(high) == null | a.getValueByOrd(high) != null); // make sure we actually can get these values
|
||||
assert low != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
assert high == BytesValues.WithOrdinals.MISSING_ORDINAL || (a.getValueByOrd(high) == null | a.getValueByOrd(high) != null); // make sure we actually can get these values
|
||||
assert low == high + 1 || a.getValueByOrd(low) == null | a.getValueByOrd(low) != null;
|
||||
while (low <= high) {
|
||||
long mid = (low + high) >>> 1;
|
||||
|
@ -386,15 +381,15 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
|
|||
return -(low + 1);
|
||||
}
|
||||
|
||||
static long getRelevantOrd(Ordinals.Docs readerOrds, int docId, MultiValueMode sortMode) {
|
||||
static long getRelevantOrd(BytesValues.WithOrdinals readerOrds, int docId, MultiValueMode sortMode) {
|
||||
int length = readerOrds.setDocument(docId);
|
||||
long relevantVal = sortMode.startLong();
|
||||
long result = Ordinals.MISSING_ORDINAL;
|
||||
long result = BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
assert sortMode == MultiValueMode.MAX || sortMode == MultiValueMode.MIN;
|
||||
for (int i = 0; i < length; i++) {
|
||||
result = relevantVal = sortMode.apply(readerOrds.nextOrd(), relevantVal);
|
||||
}
|
||||
assert result >= Ordinals.MISSING_ORDINAL;
|
||||
assert result >= BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
assert result < readerOrds.getMaxOrd();
|
||||
return result;
|
||||
// Enable this when the api can tell us that the ords per doc are ordered
|
||||
|
|
|
@ -1,95 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.util.LongsRef;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
|
||||
/**
|
||||
* Ordinals that effectively are single valued and map "one to one" to the
|
||||
* doc ids. Note, the docId is incremented by 1 to get the ordinal, since 0
|
||||
* denotes an empty value.
|
||||
*/
|
||||
public class DocIdOrdinals implements Ordinals {
|
||||
|
||||
private final int numDocs;
|
||||
|
||||
/**
|
||||
* Constructs a new doc id ordinals.
|
||||
*/
|
||||
public DocIdOrdinals(int numDocs) {
|
||||
this.numDocs = numDocs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return Ordinals.MIN_ORDINAL + numDocs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Ordinals.Docs ordinals() {
|
||||
return new Docs(this);
|
||||
}
|
||||
|
||||
public static class Docs extends Ordinals.AbstractDocs {
|
||||
|
||||
private final LongsRef longsScratch = new LongsRef(new long[1], 0, 1);
|
||||
private int docId = -1;
|
||||
private long currentOrdinal = -1;
|
||||
|
||||
public Docs(DocIdOrdinals parent) {
|
||||
super(parent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
return currentOrdinal = docId + 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
assert docId >= 0;
|
||||
currentOrdinal = docId + 1;
|
||||
docId = -1;
|
||||
return currentOrdinal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long currentOrd() {
|
||||
return currentOrdinal;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,75 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
|
||||
/**
|
||||
*/
|
||||
public enum EmptyOrdinals implements Ordinals {
|
||||
INSTANCE;
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Docs ordinals() {
|
||||
return new Docs(this);
|
||||
}
|
||||
|
||||
public static class Docs extends Ordinals.AbstractDocs {
|
||||
|
||||
public Docs(EmptyOrdinals parent) {
|
||||
super(parent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
return Ordinals.MISSING_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
throw new ElasticsearchIllegalStateException("Empty ordinals has no nextOrd");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long currentOrd() {
|
||||
return Ordinals.MISSING_ORDINAL;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
|
||||
/**
|
||||
* A {@link BytesValues.WithOrdinals} implementation that returns ordinals that are global.
|
||||
*/
|
||||
public class GlobalOrdinalMapping extends BytesValues.WithOrdinals {
|
||||
|
||||
private final BytesValues.WithOrdinals values;
|
||||
private final OrdinalMap ordinalMap;
|
||||
private final BytesValues.WithOrdinals[] bytesValues;
|
||||
private final int segmentIndex;
|
||||
|
||||
GlobalOrdinalMapping(OrdinalMap ordinalMap, BytesValues.WithOrdinals[] bytesValues, int segmentIndex) {
|
||||
super(bytesValues[segmentIndex].isMultiValued());
|
||||
this.values = bytesValues[segmentIndex];
|
||||
this.segmentIndex = segmentIndex;
|
||||
this.bytesValues = bytesValues;
|
||||
this.ordinalMap = ordinalMap;
|
||||
}
|
||||
|
||||
int readerIndex;
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
return bytesValues[readerIndex].copyShared();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return ordinalMap.getValueCount();
|
||||
}
|
||||
|
||||
public long getGlobalOrd(long segmentOrd) {
|
||||
return segmentOrd == MISSING_ORDINAL ? MISSING_ORDINAL : ordinalMap.getGlobalOrd(segmentIndex, segmentOrd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
return getGlobalOrd(values.getOrd(docId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
return getGlobalOrd(values.nextOrd());
|
||||
}
|
||||
|
||||
@Override
|
||||
public long currentOrd() {
|
||||
return getGlobalOrd(values.currentOrd());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
return values.setDocument(docId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long globalOrd) {
|
||||
final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);
|
||||
readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd);
|
||||
return bytesValues[readerIndex].getValueByOrd(segmentOrd);
|
||||
}
|
||||
|
||||
}
|
|
@ -19,37 +19,23 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefIterator;
|
||||
import org.apache.lucene.util.PriorityQueue;
|
||||
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
|
||||
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
|
||||
import org.apache.lucene.util.packed.PackedInts;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.AbstractIndexComponent;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class InternalGlobalOrdinalsBuilder extends AbstractIndexComponent implements GlobalOrdinalsBuilder {
|
||||
|
||||
public final static int ORDINAL_MAPPING_THRESHOLD_DEFAULT = 2048;
|
||||
public final static String ORDINAL_MAPPING_THRESHOLD_KEY = "global_ordinals_compress_threshold";
|
||||
public final static String ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY = "index." + ORDINAL_MAPPING_THRESHOLD_KEY;
|
||||
|
||||
public InternalGlobalOrdinalsBuilder(Index index, @IndexSettings Settings indexSettings) {
|
||||
super(index, indexSettings);
|
||||
}
|
||||
|
@ -59,367 +45,27 @@ public class InternalGlobalOrdinalsBuilder extends AbstractIndexComponent implem
|
|||
assert indexReader.leaves().size() > 1;
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// It makes sense to make the overhead ratio configurable for the mapping from segment ords to global ords
|
||||
// However, other mappings are never the bottleneck and only used to get the original value from an ord, so
|
||||
// it makes sense to force COMPACT for them
|
||||
final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
|
||||
final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT);
|
||||
final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer(PackedInts.COMPACT);
|
||||
|
||||
FieldDataType fieldDataType = indexFieldData.getFieldDataType();
|
||||
int defaultThreshold = settings.getAsInt(ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
|
||||
int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
|
||||
OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder(
|
||||
indexReader.leaves().size(), acceptableOverheadRatio, threshold
|
||||
);
|
||||
|
||||
long currentGlobalOrdinal = 0;
|
||||
final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
|
||||
TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
|
||||
for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
|
||||
globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
|
||||
long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
|
||||
globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
|
||||
for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
|
||||
ordinalMappingBuilder.onOrdinal(leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
|
||||
}
|
||||
currentGlobalOrdinal++;
|
||||
final AtomicFieldData.WithOrdinals<?>[] atomicFD = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
|
||||
final TermsEnum[] subs = new TermsEnum[indexReader.leaves().size()];
|
||||
for (int i = 0; i < indexReader.leaves().size(); ++i) {
|
||||
atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i));
|
||||
subs[i] = atomicFD[i].getTermsEnum();
|
||||
}
|
||||
|
||||
// ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
|
||||
long memorySizeInBytesCounter = 0;
|
||||
globalOrdToFirstSegment.freeze();
|
||||
memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
|
||||
globalOrdToFirstSegmentDelta.freeze();
|
||||
memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();
|
||||
|
||||
final long maxOrd = currentGlobalOrdinal;
|
||||
OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
|
||||
// add ram used for the main segmentOrd to globalOrd lookups
|
||||
memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();
|
||||
|
||||
final long memorySizeInBytes = memorySizeInBytesCounter;
|
||||
final OrdinalMap ordinalMap = new OrdinalMap(null, subs);
|
||||
final long memorySizeInBytes = ordinalMap.ramBytesUsed();
|
||||
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
// this does include the [] from the array in the impl name
|
||||
String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
|
||||
logger.debug(
|
||||
"Global-ordinals[{}][{}][{}] took {} ms",
|
||||
implName,
|
||||
"Global-ordinals[{}][{}] took {} ms",
|
||||
indexFieldData.getFieldNames().fullName(),
|
||||
maxOrd,
|
||||
ordinalMap.getValueCount(),
|
||||
(System.currentTimeMillis() - startTime)
|
||||
);
|
||||
}
|
||||
return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(),
|
||||
fieldDataType, withOrdinals, globalOrdToFirstSegment, globalOrdToFirstSegmentDelta,
|
||||
segmentOrdToGlobalOrdLookups, memorySizeInBytes
|
||||
indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes
|
||||
);
|
||||
}
|
||||
|
||||
public interface OrdinalMappingSource {
|
||||
|
||||
Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals);
|
||||
|
||||
}
|
||||
|
||||
public static abstract class GlobalOrdinalMapping implements Ordinals.Docs {
|
||||
|
||||
protected final Ordinals.Docs segmentOrdinals;
|
||||
private final long memorySizeInBytes;
|
||||
protected final long maxOrd;
|
||||
|
||||
protected long currentGlobalOrd;
|
||||
|
||||
private GlobalOrdinalMapping(Ordinals.Docs segmentOrdinals, long memorySizeInBytes, long maxOrd) {
|
||||
this.segmentOrdinals = segmentOrdinals;
|
||||
this.memorySizeInBytes = memorySizeInBytes;
|
||||
this.maxOrd = maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final boolean isMultiValued() {
|
||||
return segmentOrdinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public final int setDocument(int docId) {
|
||||
return segmentOrdinals.setDocument(docId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long currentOrd() {
|
||||
return currentGlobalOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long getOrd(int docId) {
|
||||
long segmentOrd = segmentOrdinals.getOrd(docId);
|
||||
if (segmentOrd == Ordinals.MISSING_ORDINAL) {
|
||||
return currentGlobalOrd = Ordinals.MISSING_ORDINAL;
|
||||
} else {
|
||||
return currentGlobalOrd = getGlobalOrd(segmentOrd);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long nextOrd() {
|
||||
long segmentOrd = segmentOrdinals.nextOrd();
|
||||
return currentGlobalOrd = getGlobalOrd(segmentOrd);
|
||||
}
|
||||
|
||||
public abstract long getGlobalOrd(long segmentOrd);
|
||||
|
||||
}
|
||||
|
||||
private final static class OrdinalMappingSourceBuilder {
|
||||
|
||||
final MonotonicAppendingLongBuffer[] segmentOrdToGlobalOrdDeltas;
|
||||
final float acceptableOverheadRatio;
|
||||
final int numSegments;
|
||||
final int threshold;
|
||||
|
||||
long memorySizeInBytesCounter;
|
||||
|
||||
private OrdinalMappingSourceBuilder(int numSegments, float acceptableOverheadRatio, int threshold) {
|
||||
segmentOrdToGlobalOrdDeltas = new MonotonicAppendingLongBuffer[numSegments];
|
||||
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
|
||||
segmentOrdToGlobalOrdDeltas[i] = new MonotonicAppendingLongBuffer(acceptableOverheadRatio);
|
||||
}
|
||||
this.numSegments = numSegments;
|
||||
this.acceptableOverheadRatio = acceptableOverheadRatio;
|
||||
this.threshold = threshold;
|
||||
}
|
||||
|
||||
public void onOrdinal(int readerIndex, long segmentOrdinal, long globalOrdinal) {
|
||||
long delta = globalOrdinal - segmentOrdinal;
|
||||
segmentOrdToGlobalOrdDeltas[readerIndex].add(delta);
|
||||
}
|
||||
|
||||
public OrdinalMappingSource[] build(long maxOrd) {
|
||||
// If we find out that there are less then predefined number of ordinals, it is better to put the the
|
||||
// segment ordinal to global ordinal mapping in a packed ints, since the amount values are small and
|
||||
// will most likely fit in the CPU caches and MonotonicAppendingLongBuffer's compression will just be
|
||||
// unnecessary.
|
||||
|
||||
if (maxOrd <= threshold) {
|
||||
// Rebuilding from MonotonicAppendingLongBuffer to PackedInts.Mutable is fast
|
||||
PackedInts.Mutable[] newSegmentOrdToGlobalOrdDeltas = new PackedInts.Mutable[numSegments];
|
||||
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
|
||||
newSegmentOrdToGlobalOrdDeltas[i] = PackedInts.getMutable((int) segmentOrdToGlobalOrdDeltas[i].size(), PackedInts.bitsRequired(maxOrd), acceptableOverheadRatio);
|
||||
}
|
||||
|
||||
for (int readerIndex = 0; readerIndex < segmentOrdToGlobalOrdDeltas.length; readerIndex++) {
|
||||
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdDelta = segmentOrdToGlobalOrdDeltas[readerIndex];
|
||||
|
||||
for (long ordIndex = 0; ordIndex < segmentOrdToGlobalOrdDelta.size(); ordIndex++) {
|
||||
long ordDelta = segmentOrdToGlobalOrdDelta.get(ordIndex);
|
||||
newSegmentOrdToGlobalOrdDeltas[readerIndex].set((int) ordIndex, ordDelta);
|
||||
}
|
||||
}
|
||||
|
||||
PackedIntOrdinalMappingSource[] sources = new PackedIntOrdinalMappingSource[numSegments];
|
||||
for (int i = 0; i < newSegmentOrdToGlobalOrdDeltas.length; i++) {
|
||||
PackedInts.Reader segmentOrdToGlobalOrdDelta = newSegmentOrdToGlobalOrdDeltas[i];
|
||||
if (segmentOrdToGlobalOrdDelta.size() == maxOrd) {
|
||||
// This means that a segment contains all the value and in that case segment ordinals
|
||||
// can be used as global ordinals. This will save an extra lookup per hit.
|
||||
sources[i] = null;
|
||||
} else {
|
||||
long ramUsed = segmentOrdToGlobalOrdDelta.ramBytesUsed();
|
||||
sources[i] = new PackedIntOrdinalMappingSource(segmentOrdToGlobalOrdDelta, ramUsed, maxOrd);
|
||||
memorySizeInBytesCounter += ramUsed;
|
||||
}
|
||||
|
||||
}
|
||||
return sources;
|
||||
} else {
|
||||
OrdinalMappingSource[] sources = new OrdinalMappingSource[segmentOrdToGlobalOrdDeltas.length];
|
||||
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
|
||||
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdDeltas[i];
|
||||
if (segmentOrdToGlobalOrdLookup.size() == maxOrd) {
|
||||
// idem as above
|
||||
sources[i] = null;
|
||||
} else {
|
||||
segmentOrdToGlobalOrdLookup.freeze();
|
||||
long ramUsed = segmentOrdToGlobalOrdLookup.ramBytesUsed();
|
||||
sources[i] = new CompressedOrdinalMappingSource(segmentOrdToGlobalOrdLookup, ramUsed, maxOrd);
|
||||
memorySizeInBytesCounter += ramUsed;
|
||||
}
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
}
|
||||
|
||||
public long getMemorySizeInBytes() {
|
||||
return memorySizeInBytesCounter;
|
||||
}
|
||||
}
|
||||
|
||||
private final static class CompressedOrdinalMappingSource implements OrdinalMappingSource {
|
||||
|
||||
private final MonotonicAppendingLongBuffer globalOrdinalMapping;
|
||||
private final long memorySizeInBytes;
|
||||
private final long maxOrd;
|
||||
|
||||
private CompressedOrdinalMappingSource(MonotonicAppendingLongBuffer globalOrdinalMapping, long memorySizeInBytes, long maxOrd) {
|
||||
this.globalOrdinalMapping = globalOrdinalMapping;
|
||||
this.memorySizeInBytes = memorySizeInBytes;
|
||||
this.maxOrd = maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals) {
|
||||
return new GlobalOrdinalsDocs(segmentOrdinals, globalOrdinalMapping, memorySizeInBytes, maxOrd);
|
||||
}
|
||||
|
||||
private final static class GlobalOrdinalsDocs extends GlobalOrdinalMapping {
|
||||
|
||||
private final MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup;
|
||||
|
||||
private GlobalOrdinalsDocs(Ordinals.Docs segmentOrdinals, MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup, long memorySizeInBytes, long maxOrd) {
|
||||
super(segmentOrdinals, memorySizeInBytes, maxOrd);
|
||||
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getGlobalOrd(long segmentOrd) {
|
||||
return segmentOrd + segmentOrdToGlobalOrdLookup.get(segmentOrd);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static final class PackedIntOrdinalMappingSource implements OrdinalMappingSource {
|
||||
|
||||
private final PackedInts.Reader segmentOrdToGlobalOrdLookup;
|
||||
private final long memorySizeInBytes;
|
||||
private final long maxOrd;
|
||||
|
||||
private PackedIntOrdinalMappingSource(PackedInts.Reader segmentOrdToGlobalOrdLookup, long memorySizeInBytes, long maxOrd) {
|
||||
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
|
||||
this.memorySizeInBytes = memorySizeInBytes;
|
||||
this.maxOrd = maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Ordinals.Docs globalOrdinals(Ordinals.Docs segmentOrdinals) {
|
||||
return new GlobalOrdinalsDocs(segmentOrdinals, memorySizeInBytes, maxOrd, segmentOrdToGlobalOrdLookup);
|
||||
}
|
||||
|
||||
private final static class GlobalOrdinalsDocs extends GlobalOrdinalMapping {
|
||||
|
||||
private final PackedInts.Reader segmentOrdToGlobalOrdLookup;
|
||||
|
||||
private GlobalOrdinalsDocs(Ordinals.Docs segmentOrdinals, long memorySizeInBytes, long maxOrd, PackedInts.Reader segmentOrdToGlobalOrdLookup) {
|
||||
super(segmentOrdinals, memorySizeInBytes, maxOrd);
|
||||
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getGlobalOrd(long segmentOrd) {
|
||||
return segmentOrd + segmentOrdToGlobalOrdLookup.get((int) segmentOrd);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final static class TermIterator implements BytesRefIterator {
|
||||
|
||||
private final LeafSourceQueue sources;
|
||||
private final List<LeafSource> competitiveLeafs = new ArrayList<>();
|
||||
|
||||
private TermIterator(IndexFieldData.WithOrdinals indexFieldData, List<AtomicReaderContext> leaves, AtomicFieldData.WithOrdinals[] withOrdinals) throws IOException {
|
||||
this.sources = new LeafSourceQueue(leaves.size());
|
||||
for (int i = 0; i < leaves.size(); i++) {
|
||||
AtomicReaderContext atomicReaderContext = leaves.get(i);
|
||||
AtomicFieldData.WithOrdinals afd = indexFieldData.load(atomicReaderContext);
|
||||
withOrdinals[i] = afd;
|
||||
LeafSource leafSource = new LeafSource(afd, atomicReaderContext);
|
||||
if (leafSource.current != null) {
|
||||
sources.add(leafSource);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public BytesRef next() throws IOException {
|
||||
for (LeafSource top : competitiveLeafs) {
|
||||
if (top.next() != null) {
|
||||
sources.add(top);
|
||||
}
|
||||
}
|
||||
competitiveLeafs.clear();
|
||||
if (sources.size() == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
do {
|
||||
LeafSource competitiveLeaf = sources.pop();
|
||||
competitiveLeafs.add(competitiveLeaf);
|
||||
} while (sources.size() > 0 && competitiveLeafs.get(0).current.equals(sources.top().current));
|
||||
return competitiveLeafs.get(0).current;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<BytesRef> getComparator() {
|
||||
return BytesRef.getUTF8SortedAsUnicodeComparator();
|
||||
}
|
||||
|
||||
List<LeafSource> competitiveLeafs() throws IOException {
|
||||
return competitiveLeafs;
|
||||
}
|
||||
|
||||
int firstReaderIndex() {
|
||||
return competitiveLeafs.get(0).context.ord;
|
||||
}
|
||||
|
||||
long firstLocalOrdinal() throws IOException {
|
||||
return competitiveLeafs.get(0).tenum.ord();
|
||||
}
|
||||
|
||||
private static class LeafSource {
|
||||
|
||||
final TermsEnum tenum;
|
||||
final AtomicReaderContext context;
|
||||
|
||||
BytesRef current;
|
||||
|
||||
private LeafSource(AtomicFieldData.WithOrdinals afd, AtomicReaderContext context) throws IOException {
|
||||
this.tenum = afd.getTermsEnum();
|
||||
this.context = context;
|
||||
this.current = tenum.next();
|
||||
}
|
||||
|
||||
BytesRef next() throws IOException {
|
||||
return current = tenum.next();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final static class LeafSourceQueue extends PriorityQueue<LeafSource> {
|
||||
|
||||
private final Comparator<BytesRef> termComp = BytesRef.getUTF8SortedAsUnicodeComparator();
|
||||
|
||||
LeafSourceQueue(int size) {
|
||||
super(size);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean lessThan(LeafSource termsA, LeafSource termsB) {
|
||||
final int cmp = termComp.compare(termsA.current, termsB.current);
|
||||
if (cmp != 0) {
|
||||
return cmp < 0;
|
||||
} else {
|
||||
return termsA.context.ord < termsB.context.ord;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,16 +19,14 @@
|
|||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.LongValues;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.OrdinalMappingSource;
|
||||
import org.elasticsearch.index.fielddata.plain.AtomicFieldDataWithOrdinalsTermsEnum;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
|
||||
|
@ -39,11 +37,11 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
|
|||
|
||||
private final Atomic[] atomicReaders;
|
||||
|
||||
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData.WithOrdinals[] segmentAfd, LongValues globalOrdToFirstSegment, LongValues globalOrdToFirstSegmentDelta, OrdinalMappingSource[] segmentOrdToGlobalOrds, long memorySizeInBytes) {
|
||||
InternalGlobalOrdinalsIndexFieldData(Index index, Settings settings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData.WithOrdinals[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) {
|
||||
super(index, settings, fieldNames, fieldDataType, memorySizeInBytes);
|
||||
this.atomicReaders = new Atomic[segmentAfd.length];
|
||||
for (int i = 0; i < segmentAfd.length; i++) {
|
||||
atomicReaders[i] = new Atomic(segmentAfd[i], globalOrdToFirstSegment, globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrds[i]);
|
||||
atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -55,57 +53,27 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
|
|||
private final class Atomic implements AtomicFieldData.WithOrdinals {
|
||||
|
||||
private final WithOrdinals afd;
|
||||
private final OrdinalMappingSource segmentOrdToGlobalOrdLookup;
|
||||
private final LongValues globalOrdToFirstSegment;
|
||||
private final LongValues globalOrdToFirstSegmentDelta;
|
||||
private final OrdinalMap ordinalMap;
|
||||
private final int segmentIndex;
|
||||
|
||||
private Atomic(WithOrdinals afd, LongValues globalOrdToFirstSegment, LongValues globalOrdToFirstSegmentDelta, OrdinalMappingSource segmentOrdToGlobalOrdLookup) {
|
||||
private Atomic(WithOrdinals afd, OrdinalMap ordinalMap, int segmentIndex) {
|
||||
this.afd = afd;
|
||||
this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
|
||||
this.globalOrdToFirstSegment = globalOrdToFirstSegment;
|
||||
this.globalOrdToFirstSegmentDelta = globalOrdToFirstSegmentDelta;
|
||||
this.ordinalMap = ordinalMap;
|
||||
this.segmentIndex = segmentIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
BytesValues.WithOrdinals values = afd.getBytesValues();
|
||||
Ordinals.Docs segmentOrdinals = values.ordinals();
|
||||
final Ordinals.Docs globalOrdinals;
|
||||
if (segmentOrdToGlobalOrdLookup != null) {
|
||||
globalOrdinals = segmentOrdToGlobalOrdLookup.globalOrdinals(segmentOrdinals);
|
||||
} else {
|
||||
globalOrdinals = segmentOrdinals;
|
||||
final BytesValues.WithOrdinals values = afd.getBytesValues();
|
||||
if (values.getMaxOrd() == ordinalMap.getValueCount()) {
|
||||
// segment ordinals match global ordinals
|
||||
return values;
|
||||
}
|
||||
final BytesValues.WithOrdinals[] bytesValues = new BytesValues.WithOrdinals[atomicReaders.length];
|
||||
for (int i = 0; i < bytesValues.length; i++) {
|
||||
bytesValues[i] = atomicReaders[i].afd.getBytesValues();
|
||||
}
|
||||
return new BytesValues.WithOrdinals(globalOrdinals) {
|
||||
|
||||
int readerIndex;
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long globalOrd) {
|
||||
final long segmentOrd = globalOrd - globalOrdToFirstSegmentDelta.get(globalOrd);
|
||||
readerIndex = (int) globalOrdToFirstSegment.get(globalOrd);
|
||||
return bytesValues[readerIndex].getValueByOrd(segmentOrd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
return bytesValues[readerIndex].copyShared();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return afd.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return afd.getNumberUniqueValues();
|
||||
return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -19,15 +19,18 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.LongsRef;
|
||||
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
|
||||
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
|
||||
import org.apache.lucene.util.packed.PackedInts;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
|
||||
|
||||
/**
|
||||
* {@link Ordinals} implementation which is efficient at storing field data ordinals for multi-valued or sparse fields.
|
||||
*/
|
||||
public class MultiOrdinals implements Ordinals {
|
||||
public class MultiOrdinals extends Ordinals {
|
||||
|
||||
private static final int OFFSETS_PAGE_SIZE = 1024;
|
||||
private static final int OFFSET_INIT_PAGE_COUNT = 16;
|
||||
|
@ -74,38 +77,36 @@ public class MultiOrdinals implements Ordinals {
|
|||
assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return endOffsets.ramBytesUsed() + ords.ramBytesUsed();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return multiValued;
|
||||
public WithOrdinals ordinals(ValuesHolder values) {
|
||||
return new MultiDocs(this, values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Ordinals.Docs ordinals() {
|
||||
return new MultiDocs(this);
|
||||
}
|
||||
|
||||
static class MultiDocs extends Ordinals.AbstractDocs {
|
||||
public static class MultiDocs extends BytesValues.WithOrdinals {
|
||||
|
||||
private final long maxOrd;
|
||||
private final MonotonicAppendingLongBuffer endOffsets;
|
||||
private final AppendingPackedLongBuffer ords;
|
||||
private long offset;
|
||||
private long limit;
|
||||
private long currentOrd;
|
||||
private final ValuesHolder values;
|
||||
|
||||
MultiDocs(MultiOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
MultiDocs(MultiOrdinals ordinals, ValuesHolder values) {
|
||||
super(ordinals.multiValued);
|
||||
this.maxOrd = ordinals.maxOrd;
|
||||
this.endOffsets = ordinals.endOffsets;
|
||||
this.ords = ordinals.ords;
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -113,7 +114,7 @@ public class MultiOrdinals implements Ordinals {
|
|||
final long startOffset = docId > 0 ? endOffsets.get(docId - 1) : 0;
|
||||
final long endOffset = endOffsets.get(docId);
|
||||
if (startOffset == endOffset) {
|
||||
return currentOrd = Ordinals.MISSING_ORDINAL; // ord for missing values
|
||||
return currentOrd = MISSING_ORDINAL; // ord for missing values
|
||||
} else {
|
||||
return currentOrd = ords.get(startOffset);
|
||||
}
|
||||
|
@ -138,5 +139,15 @@ public class MultiOrdinals implements Ordinals {
|
|||
public long currentOrd() {
|
||||
return currentOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
return values.getValueByOrd(ord);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
return values.copy(scratch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,124 +19,37 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.index.SortedSetDocValues;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* A thread safe ordinals abstraction. Ordinals can only be positive integers.
|
||||
*/
|
||||
public interface Ordinals {
|
||||
public abstract class Ordinals {
|
||||
|
||||
static final long MISSING_ORDINAL = SortedSetDocValues.NO_MORE_ORDS;
|
||||
static final long MIN_ORDINAL = 0;
|
||||
public static final ValuesHolder NO_VALUES = new ValuesHolder() {
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* The memory size this ordinals take.
|
||||
*/
|
||||
long getMemorySizeInBytes();
|
||||
public abstract long getMemorySizeInBytes();
|
||||
|
||||
/**
|
||||
* Is one of the docs maps to more than one ordinal?
|
||||
*/
|
||||
boolean isMultiValued();
|
||||
public abstract BytesValues.WithOrdinals ordinals(ValuesHolder values);
|
||||
|
||||
/**
|
||||
* Returns total unique ord count; this includes +1 for
|
||||
* the {@link #MISSING_ORDINAL} ord (always {@value #MISSING_ORDINAL} ).
|
||||
*/
|
||||
long getMaxOrd();
|
||||
|
||||
/**
|
||||
* Returns a lightweight (non thread safe) view iterator of the ordinals.
|
||||
*/
|
||||
Docs ordinals();
|
||||
|
||||
/**
|
||||
* A non thread safe ordinals abstraction, yet very lightweight to create. The idea
|
||||
* is that this gets created for each "iteration" over ordinals.
|
||||
* <p/>
|
||||
* <p>A value of 0 ordinal when iterating indicated "no" value.</p>
|
||||
* To iterate of a set of ordinals for a given document use {@link #setDocument(int)} and {@link #nextOrd()} as
|
||||
* show in the example below:
|
||||
* <pre>
|
||||
* Ordinals.Docs docs = ...;
|
||||
* final int len = docs.setDocId(docId);
|
||||
* for (int i = 0; i < len; i++) {
|
||||
* final long ord = docs.nextOrd();
|
||||
* // process ord
|
||||
* }
|
||||
* </pre>
|
||||
*/
|
||||
interface Docs {
|
||||
|
||||
/**
|
||||
* Returns total unique ord count; this includes +1 for
|
||||
* the null ord (always 0).
|
||||
*/
|
||||
long getMaxOrd();
|
||||
|
||||
/**
|
||||
* Is one of the docs maps to more than one ordinal?
|
||||
*/
|
||||
boolean isMultiValued();
|
||||
|
||||
/**
|
||||
* The ordinal that maps to the relevant docId. If it has no value, returns
|
||||
* <tt>0</tt>.
|
||||
*/
|
||||
long getOrd(int docId);
|
||||
|
||||
|
||||
/**
|
||||
* Returns the next ordinal for the current docID set to {@link #setDocument(int)}.
|
||||
* This method should only be called <tt>N</tt> times where <tt>N</tt> is the number
|
||||
* returned from {@link #setDocument(int)}. If called more than <tt>N</tt> times the behavior
|
||||
* is undefined.
|
||||
*
|
||||
* Note: This method will never return <tt>0</tt>.
|
||||
*
|
||||
* @return the next ordinal for the current docID set to {@link #setDocument(int)}.
|
||||
*/
|
||||
long nextOrd();
|
||||
|
||||
|
||||
/**
|
||||
* Sets iteration to the specified docID and returns the number of
|
||||
* ordinals for this document ID,
|
||||
* @param docId document ID
|
||||
*
|
||||
* @see #nextOrd()
|
||||
*/
|
||||
int setDocument(int docId);
|
||||
|
||||
|
||||
/**
|
||||
* Returns the current ordinal in the iteration
|
||||
* @return the current ordinal in the iteration
|
||||
*/
|
||||
long currentOrd();
|
||||
public final BytesValues.WithOrdinals ordinals() {
|
||||
return ordinals(NO_VALUES);
|
||||
}
|
||||
|
||||
/**
|
||||
* Base implementation of {@link Docs}.
|
||||
*/
|
||||
public static abstract class AbstractDocs implements Docs {
|
||||
public static interface ValuesHolder {
|
||||
|
||||
protected final Ordinals ordinals;
|
||||
|
||||
public AbstractDocs(Ordinals ordinals) {
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long getMaxOrd() {
|
||||
return ordinals.getMaxOrd();
|
||||
}
|
||||
|
||||
@Override
|
||||
public final boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
public abstract BytesRef getValueByOrd(long ord);
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -27,6 +27,7 @@ import org.apache.lucene.util.packed.GrowableWriter;
|
|||
import org.apache.lucene.util.packed.PackedInts;
|
||||
import org.apache.lucene.util.packed.PagedGrowableWriter;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
|
@ -259,7 +260,7 @@ public final class OrdinalsBuilder implements Closeable {
|
|||
}
|
||||
|
||||
private final int maxDoc;
|
||||
private long currentOrd = Ordinals.MIN_ORDINAL - 1;
|
||||
private long currentOrd = BytesValues.WithOrdinals.MIN_ORDINAL - 1;
|
||||
private int numDocsWithValue = 0;
|
||||
private int numMultiValuedDocs = 0;
|
||||
private int totalNumOrds = 0;
|
||||
|
|
|
@ -19,12 +19,15 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
import org.apache.lucene.util.packed.PackedInts;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.BytesValues.WithOrdinals;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class SinglePackedOrdinals implements Ordinals {
|
||||
public class SinglePackedOrdinals extends Ordinals {
|
||||
|
||||
// ordinals with value 0 indicates no value
|
||||
private final PackedInts.Reader reader;
|
||||
|
@ -36,7 +39,7 @@ public class SinglePackedOrdinals implements Ordinals {
|
|||
assert builder.getNumMultiValuesDocs() == 0;
|
||||
this.maxOrd = builder.getMaxOrd();
|
||||
// We don't reuse the builder as-is because it might have been built with a higher overhead ratio
|
||||
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(getMaxOrd()), acceptableOverheadRatio);
|
||||
final PackedInts.Mutable reader = PackedInts.getMutable(builder.maxDoc(), PackedInts.bitsRequired(maxOrd), acceptableOverheadRatio);
|
||||
PackedInts.copy(builder.getFirstOrdinals(), 0, reader, 0, builder.maxDoc(), 8 * 1024);
|
||||
this.reader = reader;
|
||||
}
|
||||
|
@ -50,29 +53,28 @@ public class SinglePackedOrdinals implements Ordinals {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
public WithOrdinals ordinals(ValuesHolder values) {
|
||||
return new Docs(this, values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Docs ordinals() {
|
||||
return new Docs(this, reader);
|
||||
}
|
||||
|
||||
public static class Docs extends Ordinals.AbstractDocs {
|
||||
private static class Docs extends BytesValues.WithOrdinals {
|
||||
|
||||
private final long maxOrd;
|
||||
private final PackedInts.Reader reader;
|
||||
private final ValuesHolder values;
|
||||
|
||||
private long currentOrdinal;
|
||||
|
||||
public Docs(SinglePackedOrdinals parent, PackedInts.Reader reader) {
|
||||
super(parent);
|
||||
this.reader = reader;
|
||||
public Docs(SinglePackedOrdinals parent, ValuesHolder values) {
|
||||
super(false);
|
||||
this.maxOrd = parent.maxOrd;
|
||||
this.reader = parent.reader;
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -82,7 +84,7 @@ public class SinglePackedOrdinals implements Ordinals {
|
|||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
assert currentOrdinal >= Ordinals.MIN_ORDINAL;
|
||||
assert currentOrdinal >= MIN_ORDINAL;
|
||||
return currentOrdinal;
|
||||
}
|
||||
|
||||
|
@ -97,5 +99,15 @@ public class SinglePackedOrdinals implements Ordinals {
|
|||
public long currentOrd() {
|
||||
return currentOrdinal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
return values.getValueByOrd(ord);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
return values.copy(scratch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -38,16 +38,6 @@ abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<Ato
|
|||
|
||||
protected static class Empty extends AtomicGeoPointFieldData<ScriptDocValues> {
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
|
|
|
@ -26,7 +26,6 @@ import org.apache.lucene.util.Bits;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
|
@ -38,16 +37,14 @@ import java.util.Comparator;
|
|||
public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
|
||||
|
||||
private final BytesValues.WithOrdinals bytesValues;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final long maxOrd;
|
||||
|
||||
private long currentOrd = Ordinals.MISSING_ORDINAL;
|
||||
private long currentOrd = BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
private BytesRef currentTerm;
|
||||
|
||||
public AtomicFieldDataWithOrdinalsTermsEnum(AtomicFieldData.WithOrdinals afd) {
|
||||
this.bytesValues = afd.getBytesValues();
|
||||
this.ordinals = bytesValues.ordinals();
|
||||
this.maxOrd = ordinals.getMaxOrd();
|
||||
this.maxOrd = bytesValues.getMaxOrd();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -70,9 +67,9 @@ public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
|
|||
|
||||
@Override
|
||||
public void seekExact(long ord) throws IOException {
|
||||
assert ord >= 0 && ord < ordinals.getMaxOrd();
|
||||
assert ord >= 0 && ord < bytesValues.getMaxOrd();
|
||||
currentOrd = ord;
|
||||
if (currentOrd == Ordinals.MISSING_ORDINAL) {
|
||||
if (currentOrd == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
currentTerm = null;
|
||||
} else {
|
||||
currentTerm = bytesValues.getValueByOrd(currentOrd);
|
||||
|
@ -125,7 +122,7 @@ public class AtomicFieldDataWithOrdinalsTermsEnum extends TermsEnum {
|
|||
|
||||
final private static long binarySearch(BytesValues.WithOrdinals a, BytesRef key) {
|
||||
long low = 1;
|
||||
long high = a.ordinals().getMaxOrd();
|
||||
long high = a.getMaxOrd();
|
||||
while (low <= high) {
|
||||
long mid = (low + high) >>> 1;
|
||||
BytesRef midVal = a.getValueByOrd(mid);
|
||||
|
|
|
@ -43,17 +43,6 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
|
|||
this.field = field;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
// probably not accurate, but a good upper limit
|
||||
return reader.maxDoc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
// TODO: Lucene doesn't expose it right now
|
||||
|
@ -81,6 +70,9 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
|
|||
|
||||
return new BytesValues(false) {
|
||||
|
||||
final BytesRef scratch = new BytesRef();
|
||||
int docId;
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
|
|
|
@ -137,16 +137,6 @@ final class BinaryDVNumericAtomicFieldData extends AbstractAtomicNumericFieldDat
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true; // no way to know
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return Long.MAX_VALUE; // no clue
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return -1; // Lucene doesn't expose it
|
||||
|
|
|
@ -36,16 +36,6 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocVal
|
|||
this.values = values == null ? DocValues.EMPTY_BINARY : values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return -1; // not exposed by Lucene
|
||||
|
@ -56,6 +46,7 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocVal
|
|||
return new BytesValues(true) {
|
||||
|
||||
final BytesRef bytes = new BytesRef();
|
||||
final BytesRef scratch = new BytesRef();
|
||||
final ByteArrayDataInput in = new ByteArrayDataInput();
|
||||
|
||||
@Override
|
||||
|
|
|
@ -59,16 +59,6 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
return DoubleValues.EMPTY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
|
@ -96,16 +86,6 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -130,14 +110,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final DoubleArray values;
|
||||
|
||||
LongValues(DoubleArray values, Ordinals.Docs ordinals) {
|
||||
LongValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final long getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
return (long) values.get(ord);
|
||||
}
|
||||
}
|
||||
|
@ -146,14 +126,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final DoubleArray values;
|
||||
|
||||
DoubleValues(DoubleArray values, Ordinals.Docs ordinals) {
|
||||
DoubleValues(DoubleArray values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
return values.get(ord);
|
||||
}
|
||||
}
|
||||
|
@ -167,23 +147,11 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final DoubleArray values;
|
||||
private final FixedBitSet set;
|
||||
private final long numOrds;
|
||||
|
||||
public SingleFixedSet(DoubleArray values, FixedBitSet set, long numOrds) {
|
||||
public SingleFixedSet(DoubleArray values, FixedBitSet set) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.set = set;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -257,26 +225,14 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
public static class Single extends DoubleArrayAtomicFieldData {
|
||||
|
||||
private final DoubleArray values;
|
||||
private final long numOrds;
|
||||
|
||||
/**
|
||||
* Note, here, we assume that there is no offset by 1 from docId, so position 0
|
||||
* is the value for docId 0.
|
||||
*/
|
||||
public Single(DoubleArray values, long numOrds) {
|
||||
public Single(DoubleArray values) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -32,7 +32,6 @@ import org.elasticsearch.index.fielddata.*;
|
|||
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
@ -101,10 +100,10 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
|
|||
}
|
||||
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
|
||||
Ordinals build = builder.build(fieldDataType.getSettings());
|
||||
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
BytesValues.WithOrdinals ordinals = build.ordinals();
|
||||
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
data = new DoubleArrayAtomicFieldData.WithOrdinals(values, build);
|
||||
} else {
|
||||
Docs ordinals = build.ordinals();
|
||||
final FixedBitSet set = builder.buildDocsWithValuesSet();
|
||||
|
||||
// there's sweet spot where due to low unique value count, using ordinals will consume less memory
|
||||
|
@ -121,15 +120,15 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
|
|||
DoubleArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(maxDoc);
|
||||
for (int i = 0; i < maxDoc; i++) {
|
||||
final long ordinal = ordinals.getOrd(i);
|
||||
if (ordinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
sValues.set(i, values.get(ordinal));
|
||||
}
|
||||
}
|
||||
assert sValues.size() == maxDoc;
|
||||
if (set == null) {
|
||||
data = new DoubleArrayAtomicFieldData.Single(sValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new DoubleArrayAtomicFieldData.Single(sValues);
|
||||
} else {
|
||||
data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, set);
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
|
|
|
@ -1,51 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.index.fielddata.plain;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
/**
|
||||
* An empty {@link org.elasticsearch.index.fielddata.BytesValues.WithOrdinals} implementation
|
||||
*/
|
||||
final class EmptyByteValuesWithOrdinals extends BytesValues.WithOrdinals {
|
||||
|
||||
EmptyByteValuesWithOrdinals(Ordinals.Docs ordinals) {
|
||||
super(ordinals);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
scratch.length = 0;
|
||||
return scratch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef nextValue() {
|
||||
throw new ElasticsearchIllegalStateException("Empty BytesValues has no next value");
|
||||
}
|
||||
|
||||
}
|
|
@ -26,8 +26,8 @@ import org.apache.lucene.util.fst.FST.Arc;
|
|||
import org.apache.lucene.util.fst.FST.BytesReader;
|
||||
import org.apache.lucene.util.fst.Util;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -36,10 +36,6 @@ import java.io.IOException;
|
|||
*/
|
||||
public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
|
||||
|
||||
public static FSTBytesAtomicFieldData empty() {
|
||||
return new Empty();
|
||||
}
|
||||
|
||||
// 0 ordinal in values means no value (its null)
|
||||
protected final Ordinals ordinals;
|
||||
|
||||
|
@ -56,16 +52,6 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
|
|||
public void close() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -79,8 +65,7 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
|
|||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
assert fst != null;
|
||||
return new BytesValues(fst, ordinals.ordinals());
|
||||
return ordinals.ordinals(new ValuesHolder(fst));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -89,32 +74,26 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
|
|||
return new ScriptDocValues.Strings(getBytesValues());
|
||||
}
|
||||
|
||||
@Override
|
||||
public TermsEnum getTermsEnum() {
|
||||
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
|
||||
}
|
||||
private static class ValuesHolder implements Ordinals.ValuesHolder {
|
||||
|
||||
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
|
||||
|
||||
protected final FST<Long> fst;
|
||||
protected final Ordinals.Docs ordinals;
|
||||
private final FST<Long> fst;
|
||||
|
||||
// per-thread resources
|
||||
private final BytesRef scratch;
|
||||
protected final BytesReader in;
|
||||
protected final Arc<Long> firstArc = new Arc<>();
|
||||
protected final Arc<Long> scratchArc = new Arc<>();
|
||||
protected final IntsRef scratchInts = new IntsRef();
|
||||
|
||||
BytesValues(FST<Long> fst, Ordinals.Docs ordinals) {
|
||||
super(ordinals);
|
||||
ValuesHolder(FST<Long> fst) {
|
||||
this.fst = fst;
|
||||
this.ordinals = ordinals;
|
||||
scratch = new BytesRef();
|
||||
in = fst.getBytesReader();
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
in.setPosition(0);
|
||||
fst.getFirstArc(firstArc);
|
||||
try {
|
||||
|
@ -127,30 +106,6 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
|
|||
}
|
||||
return scratch;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
final static class Empty extends FSTBytesAtomicFieldData {
|
||||
|
||||
Empty() {
|
||||
super(null, EmptyOrdinals.INSTANCE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScriptDocValues.Strings getScriptValues() {
|
||||
return ScriptDocValues.EMPTY_STRINGS;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -27,9 +27,7 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
|
|||
import org.apache.lucene.util.fst.Util;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
|
@ -40,14 +38,14 @@ import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
|
|||
|
||||
/**
|
||||
*/
|
||||
public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytesAtomicFieldData> {
|
||||
public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
|
||||
|
||||
private final CircuitBreakerService breakerService;
|
||||
|
||||
public static class Builder implements IndexFieldData.Builder {
|
||||
|
||||
@Override
|
||||
public IndexFieldData<FSTBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
|
||||
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
|
||||
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
|
||||
GlobalOrdinalsBuilder globalOrdinalBuilder) {
|
||||
return new FSTBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
|
||||
|
@ -61,7 +59,7 @@ public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytes
|
|||
}
|
||||
|
||||
@Override
|
||||
public FSTBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
|
||||
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
|
||||
AtomicReader reader = context.reader();
|
||||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
|
@ -69,9 +67,8 @@ public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytes
|
|||
// TODO: Use an actual estimator to estimate before loading.
|
||||
NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker());
|
||||
if (terms == null) {
|
||||
data = FSTBytesAtomicFieldData.empty();
|
||||
estimator.afterLoad(null, data.getMemorySizeInBytes());
|
||||
return data;
|
||||
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.getMemorySizeInBytes());
|
||||
return AtomicFieldData.WithOrdinals.EMPTY;
|
||||
}
|
||||
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
|
||||
org.apache.lucene.util.fst.Builder<Long> fstBuilder = new org.apache.lucene.util.fst.Builder<>(INPUT_TYPE.BYTE1, outputs);
|
||||
|
|
|
@ -58,16 +58,6 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
return DoubleValues.EMPTY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
|
@ -95,16 +85,6 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -127,14 +107,14 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
|
||||
private final FloatArray values;
|
||||
|
||||
LongValues(FloatArray values, Ordinals.Docs ordinals) {
|
||||
LongValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
return (long) values.get(ord);
|
||||
}
|
||||
}
|
||||
|
@ -143,7 +123,7 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
|
||||
private final FloatArray values;
|
||||
|
||||
DoubleValues(FloatArray values, Ordinals.Docs ordinals) {
|
||||
DoubleValues(FloatArray values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
@ -163,23 +143,11 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
|
||||
private final FloatArray values;
|
||||
private final FixedBitSet set;
|
||||
private final long numOrd;
|
||||
|
||||
public SingleFixedSet(FloatArray values, FixedBitSet set, long numOrd) {
|
||||
public SingleFixedSet(FloatArray values, FixedBitSet set) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.set = set;
|
||||
this.numOrd = numOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -255,26 +223,14 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
|
|||
public static class Single extends FloatArrayAtomicFieldData {
|
||||
|
||||
private final FloatArray values;
|
||||
private final long numOrd;
|
||||
|
||||
/**
|
||||
* Note, here, we assume that there is no offset by 1 from docId, so position 0
|
||||
* is the value for docId 0.
|
||||
*/
|
||||
public Single(FloatArray values, long numOrd) {
|
||||
public Single(FloatArray values) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.numOrd = numOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -31,7 +31,6 @@ import org.elasticsearch.index.fielddata.*;
|
|||
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
@ -99,10 +98,10 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
|
|||
}
|
||||
values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
|
||||
Ordinals build = builder.build(fieldDataType.getSettings());
|
||||
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
BytesValues.WithOrdinals ordinals = build.ordinals();
|
||||
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
data = new FloatArrayAtomicFieldData.WithOrdinals(values, build);
|
||||
} else {
|
||||
Docs ordinals = build.ordinals();
|
||||
final FixedBitSet set = builder.buildDocsWithValuesSet();
|
||||
|
||||
// there's sweet spot where due to low unique value count, using ordinals will consume less memory
|
||||
|
@ -119,15 +118,15 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
|
|||
FloatArray sValues = BigArrays.NON_RECYCLING_INSTANCE.newFloatArray(maxDoc);
|
||||
for (int i = 0; i < maxDoc; i++) {
|
||||
final long ordinal = ordinals.getOrd(i);
|
||||
if (ordinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
sValues.set(i, values.get(ordinal));
|
||||
}
|
||||
}
|
||||
assert sValues.size() == maxDoc;
|
||||
if (set == null) {
|
||||
data = new FloatArrayAtomicFieldData.Single(sValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new FloatArrayAtomicFieldData.Single(sValues);
|
||||
} else {
|
||||
data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, set);
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
|
|
|
@ -37,16 +37,6 @@ final class GeoPointBinaryDVAtomicFieldData extends AtomicGeoPointFieldData<Scri
|
|||
this.values = values == null ? DocValues.EMPTY_BINARY : values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return -1; // not exposed by Lucene
|
||||
|
|
|
@ -23,6 +23,7 @@ import org.apache.lucene.util.RamUsageEstimator;
|
|||
import org.apache.lucene.util.packed.PagedMutable;
|
||||
import org.elasticsearch.common.geo.GeoPoint;
|
||||
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.GeoPointValues;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
@ -58,16 +59,6 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
|
|||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -85,11 +76,11 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
|
|||
|
||||
private final GeoPointFieldMapper.Encoding encoding;
|
||||
private final PagedMutable lon, lat;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final BytesValues.WithOrdinals ordinals;
|
||||
|
||||
private final GeoPoint scratch = new GeoPoint();
|
||||
|
||||
GeoPointValuesWithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, Ordinals.Docs ordinals) {
|
||||
GeoPointValuesWithOrdinals(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals.isMultiValued());
|
||||
this.encoding = encoding;
|
||||
this.lon = lon;
|
||||
|
@ -119,25 +110,13 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
|
|||
private final GeoPointFieldMapper.Encoding encoding;
|
||||
private final PagedMutable lon, lat;
|
||||
private final FixedBitSet set;
|
||||
private final long numOrds;
|
||||
|
||||
public SingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set, long numOrds) {
|
||||
public SingleFixedSet(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, FixedBitSet set) {
|
||||
super();
|
||||
this.encoding = encoding;
|
||||
this.lon = lon;
|
||||
this.lat = lat;
|
||||
this.set = set;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -190,24 +169,12 @@ public abstract class GeoPointCompressedAtomicFieldData extends AtomicGeoPointFi
|
|||
|
||||
private final GeoPointFieldMapper.Encoding encoding;
|
||||
private final PagedMutable lon, lat;
|
||||
private final long numOrds;
|
||||
|
||||
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat, long numOrds) {
|
||||
public Single(GeoPointFieldMapper.Encoding encoding, PagedMutable lon, PagedMutable lat) {
|
||||
super();
|
||||
this.encoding = encoding;
|
||||
this.lon = lon;
|
||||
this.lat = lat;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -33,7 +33,6 @@ import org.elasticsearch.index.Index;
|
|||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
@ -115,21 +114,21 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
|
|||
}
|
||||
|
||||
Ordinals build = builder.build(fieldDataType.getSettings());
|
||||
if (build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
if (lat.size() != build.getMaxOrd()) {
|
||||
lat = lat.resize(build.getMaxOrd());
|
||||
lon = lon.resize(build.getMaxOrd());
|
||||
BytesValues.WithOrdinals ordinals = build.ordinals();
|
||||
if (ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
if (lat.size() != ordinals.getMaxOrd()) {
|
||||
lat = lat.resize(ordinals.getMaxOrd());
|
||||
lon = lon.resize(ordinals.getMaxOrd());
|
||||
}
|
||||
data = new GeoPointCompressedAtomicFieldData.WithOrdinals(encoding, lon, lat, build);
|
||||
} else {
|
||||
Docs ordinals = build.ordinals();
|
||||
int maxDoc = reader.maxDoc();
|
||||
PagedMutable sLat = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
|
||||
PagedMutable sLon = new PagedMutable(reader.maxDoc(), pageSize, encoding.numBitsPerCoordinate(), PackedInts.COMPACT);
|
||||
final long missing = encoding.encodeCoordinate(0);
|
||||
for (int i = 0; i < maxDoc; i++) {
|
||||
final long nativeOrdinal = ordinals.getOrd(i);
|
||||
if (nativeOrdinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
sLat.set(i, lat.get(nativeOrdinal));
|
||||
sLon.set(i, lon.get(nativeOrdinal));
|
||||
} else {
|
||||
|
@ -139,9 +138,9 @@ public class GeoPointCompressedIndexFieldData extends AbstractGeoPointIndexField
|
|||
}
|
||||
FixedBitSet set = builder.buildDocsWithValuesSet();
|
||||
if (set == null) {
|
||||
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new GeoPointCompressedAtomicFieldData.Single(encoding, sLon, sLat);
|
||||
} else {
|
||||
data = new GeoPointCompressedAtomicFieldData.SingleFixedSet(encoding, sLon, sLat, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new GeoPointCompressedAtomicFieldData.SingleFixedSet(encoding, sLon, sLat, set);
|
||||
}
|
||||
}
|
||||
success = true;
|
||||
|
|
|
@ -23,6 +23,7 @@ import org.apache.lucene.util.RamUsageEstimator;
|
|||
import org.elasticsearch.common.geo.GeoPoint;
|
||||
import org.elasticsearch.common.util.DoubleArray;
|
||||
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.GeoPointValues;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
@ -54,16 +55,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
|
|||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -80,11 +71,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
|
|||
public static class GeoPointValuesWithOrdinals extends GeoPointValues {
|
||||
|
||||
private final DoubleArray lon, lat;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final BytesValues.WithOrdinals ordinals;
|
||||
|
||||
private final GeoPoint scratch = new GeoPoint();
|
||||
|
||||
GeoPointValuesWithOrdinals(DoubleArray lon, DoubleArray lat, Ordinals.Docs ordinals) {
|
||||
GeoPointValuesWithOrdinals(DoubleArray lon, DoubleArray lat, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals.isMultiValued());
|
||||
this.lon = lon;
|
||||
this.lat = lat;
|
||||
|
@ -112,24 +103,12 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
|
|||
|
||||
private final DoubleArray lon, lat;
|
||||
private final FixedBitSet set;
|
||||
private final long numOrds;
|
||||
|
||||
public SingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set, long numOrds) {
|
||||
public SingleFixedSet(DoubleArray lon, DoubleArray lat, FixedBitSet set) {
|
||||
super();
|
||||
this.lon = lon;
|
||||
this.lat = lat;
|
||||
this.set = set;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -180,23 +159,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
|
|||
public static class Single extends GeoPointDoubleArrayAtomicFieldData {
|
||||
|
||||
private final DoubleArray lon, lat;
|
||||
private final long numOrds;
|
||||
|
||||
public Single(DoubleArray lon, DoubleArray lat, long numOrds) {
|
||||
public Single(DoubleArray lon, DoubleArray lat) {
|
||||
super();
|
||||
this.lon = lon;
|
||||
this.lat = lat;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -30,7 +30,6 @@ import org.elasticsearch.index.Index;
|
|||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
|
@ -90,23 +89,23 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractGeoPointIndexFiel
|
|||
lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms);
|
||||
|
||||
Ordinals build = builder.build(fieldDataType.getSettings());
|
||||
if (!(build.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
|
||||
Docs ordinals = build.ordinals();
|
||||
BytesValues.WithOrdinals ordinals = build.ordinals();
|
||||
if (!(ordinals.isMultiValued() || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
|
||||
int maxDoc = reader.maxDoc();
|
||||
DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
|
||||
DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
|
||||
for (int i = 0; i < maxDoc; i++) {
|
||||
long nativeOrdinal = ordinals.getOrd(i);
|
||||
if (nativeOrdinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (nativeOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
sLat.set(i, lat.get(nativeOrdinal));
|
||||
sLon.set(i, lon.get(nativeOrdinal));
|
||||
}
|
||||
}
|
||||
FixedBitSet set = builder.buildDocsWithValuesSet();
|
||||
if (set == null) {
|
||||
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat);
|
||||
} else {
|
||||
data = new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, set, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, set);
|
||||
}
|
||||
} else {
|
||||
data = new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon, lat, build);
|
||||
|
|
|
@ -28,7 +28,6 @@ import org.elasticsearch.index.Index;
|
|||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
|
||||
|
@ -46,7 +45,15 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
|
|||
|
||||
}
|
||||
|
||||
private static final Ordinals.Docs INDEX_ORDINALS = new Ordinals.Docs() {
|
||||
private static class IndexBytesValues extends BytesValues.WithOrdinals {
|
||||
|
||||
private final BytesRef scratch;
|
||||
|
||||
protected IndexBytesValues(String index) {
|
||||
super(false);
|
||||
scratch = new BytesRef();
|
||||
scratch.copyChars(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
|
@ -55,17 +62,12 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
|
|||
|
||||
@Override
|
||||
public long nextOrd() {
|
||||
return Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
return BytesValues.WithOrdinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
return Ordinals.MIN_ORDINAL;
|
||||
return BytesValues.WithOrdinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -75,15 +77,7 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
|
|||
|
||||
@Override
|
||||
public long currentOrd() {
|
||||
return Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
};
|
||||
|
||||
private static class IndexBytesValues extends BytesValues.WithOrdinals {
|
||||
|
||||
protected IndexBytesValues(String index) {
|
||||
super(INDEX_ORDINALS);
|
||||
scratch.copyChars(index);
|
||||
return BytesValues.WithOrdinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -106,16 +100,6 @@ public class IndexIndexFieldData implements IndexFieldData.WithOrdinals<AtomicFi
|
|||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new IndexBytesValues(index);
|
||||
|
|
|
@ -48,17 +48,6 @@ public class NumericDVAtomicFieldData extends AbstractAtomicNumericFieldData {
|
|||
// no-op
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
// good upper limit
|
||||
return reader.maxDoc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
// TODO: cannot be computed from Lucene
|
||||
|
|
|
@ -58,21 +58,11 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
return DoubleValues.EMPTY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues getBytesValues() {
|
||||
return BytesValues.EMPTY;
|
||||
|
@ -95,11 +85,6 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -108,11 +93,6 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LongValues getLongValues() {
|
||||
return new LongValues(values, ordinals.ordinals());
|
||||
|
@ -127,14 +107,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final MonotonicAppendingLongBuffer values;
|
||||
|
||||
LongValues(MonotonicAppendingLongBuffer values, Ordinals.Docs ordinals) {
|
||||
LongValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
return values.get(ord);
|
||||
}
|
||||
}
|
||||
|
@ -143,14 +123,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final MonotonicAppendingLongBuffer values;
|
||||
|
||||
DoubleValues(MonotonicAppendingLongBuffer values, Ordinals.Docs ordinals) {
|
||||
DoubleValues(MonotonicAppendingLongBuffer values, BytesValues.WithOrdinals ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
return values.get(ord);
|
||||
}
|
||||
|
||||
|
@ -167,24 +147,12 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
private final PackedInts.Mutable values;
|
||||
private final long minValue;
|
||||
private final long missingValue;
|
||||
private final long numOrds;
|
||||
|
||||
public SingleSparse(PackedInts.Mutable values, long minValue, long missingValue, long numOrds) {
|
||||
public SingleSparse(PackedInts.Mutable values, long minValue, long missingValue) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.minValue = minValue;
|
||||
this.missingValue = missingValue;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -263,27 +231,15 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final PackedInts.Mutable values;
|
||||
private final long minValue;
|
||||
private final long numOrds;
|
||||
|
||||
/**
|
||||
* Note, here, we assume that there is no offset by 1 from docId, so position 0
|
||||
* is the value for docId 0.
|
||||
*/
|
||||
public Single(PackedInts.Mutable values, long minValue, long numOrds) {
|
||||
public Single(PackedInts.Mutable values, long minValue) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.minValue = minValue;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -355,26 +311,14 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
public static class PagedSingle extends PackedArrayAtomicFieldData {
|
||||
|
||||
private final AppendingDeltaPackedLongBuffer values;
|
||||
private final long numOrds;
|
||||
|
||||
/**
|
||||
* Note, here, we assume that there is no offset by 1 from docId, so position 0
|
||||
* is the value for docId 0.
|
||||
*/
|
||||
public PagedSingle(AppendingDeltaPackedLongBuffer values, long numOrds) {
|
||||
public PagedSingle(AppendingDeltaPackedLongBuffer values) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -446,23 +390,11 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
|
|||
|
||||
private final AppendingDeltaPackedLongBuffer values;
|
||||
private final FixedBitSet docsWithValue;
|
||||
private final long numOrds;
|
||||
|
||||
public PagedSingleSparse(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue, long numOrds) {
|
||||
public PagedSingleSparse(AppendingDeltaPackedLongBuffer values, FixedBitSet docsWithValue) {
|
||||
super();
|
||||
this.values = values;
|
||||
this.docsWithValue = docsWithValue;
|
||||
this.numOrds = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -35,15 +35,14 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
|
||||
import org.elasticsearch.search.MultiValueMode;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
|
||||
import org.elasticsearch.search.MultiValueMode;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.EnumSet;
|
||||
|
@ -128,10 +127,10 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
|||
Ordinals build = builder.build(fieldDataType.getSettings());
|
||||
CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType);
|
||||
|
||||
if (build.isMultiValued() || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
BytesValues.WithOrdinals ordinals = build.ordinals();
|
||||
if (ordinals.isMultiValued() || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
|
||||
data = new PackedArrayAtomicFieldData.WithOrdinals(values, build);
|
||||
} else {
|
||||
Docs ordinals = build.ordinals();
|
||||
final FixedBitSet docsWithValues = builder.buildDocsWithValuesSet();
|
||||
|
||||
long minValue, maxValue;
|
||||
|
@ -185,15 +184,15 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
|||
|
||||
for (int i = 0; i < reader.maxDoc(); i++) {
|
||||
final long ord = ordinals.getOrd(i);
|
||||
if (ord != Ordinals.MISSING_ORDINAL) {
|
||||
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
long value = values.get(ord);
|
||||
sValues.set(i, value - minValue);
|
||||
}
|
||||
}
|
||||
if (docsWithValues == null) {
|
||||
data = new PackedArrayAtomicFieldData.Single(sValues, minValue, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new PackedArrayAtomicFieldData.Single(sValues, minValue);
|
||||
} else {
|
||||
data = new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, missingValue, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, missingValue);
|
||||
}
|
||||
break;
|
||||
case PAGED:
|
||||
|
@ -203,16 +202,16 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
|||
long lastValue = 0;
|
||||
for (int i = 0; i < reader.maxDoc(); i++) {
|
||||
final long ord = ordinals.getOrd(i);
|
||||
if (ord != Ordinals.MISSING_ORDINAL) {
|
||||
if (ord != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
lastValue = values.get(ord);
|
||||
}
|
||||
dpValues.add(lastValue);
|
||||
}
|
||||
dpValues.freeze();
|
||||
if (docsWithValues == null) {
|
||||
data = new PackedArrayAtomicFieldData.PagedSingle(dpValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new PackedArrayAtomicFieldData.PagedSingle(dpValues);
|
||||
} else {
|
||||
data = new PackedArrayAtomicFieldData.PagedSingleSparse(dpValues, docsWithValues, ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL);
|
||||
data = new PackedArrayAtomicFieldData.PagedSingleSparse(dpValues, docsWithValues);
|
||||
}
|
||||
break;
|
||||
case ORDINALS:
|
||||
|
@ -239,7 +238,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
|||
|
||||
}
|
||||
|
||||
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, Docs ordinals,
|
||||
protected CommonSettings.MemoryStorageFormat chooseStorageFormat(AtomicReader reader, MonotonicAppendingLongBuffer values, Ordinals build, BytesValues.WithOrdinals ordinals,
|
||||
long minValue, long maxValue, float acceptableOverheadRatio, int pageSize) {
|
||||
|
||||
CommonSettings.MemoryStorageFormat format;
|
||||
|
@ -261,7 +260,7 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
|
|||
long pageMaxOrdinal = Long.MIN_VALUE;
|
||||
for (int i = 1; i < reader.maxDoc(); ++i, pageIndex = (pageIndex + 1) % pageSize) {
|
||||
long ordinal = ordinals.getOrd(i);
|
||||
if (ordinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (ordinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
pageMaxOrdinal = Math.max(ordinal, pageMaxOrdinal);
|
||||
pageMinOrdinal = Math.min(ordinal, pageMinOrdinal);
|
||||
}
|
||||
|
|
|
@ -23,18 +23,14 @@ import org.apache.lucene.util.BytesRef;
|
|||
import org.apache.lucene.util.PagedBytes;
|
||||
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
|
||||
|
||||
public static PagedBytesAtomicFieldData empty() {
|
||||
return new Empty();
|
||||
}
|
||||
|
||||
private final PagedBytes.Reader bytes;
|
||||
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
|
||||
protected final Ordinals ordinals;
|
||||
|
@ -53,16 +49,6 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
|
|||
public void close() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return ordinals.getMaxOrd() - Ordinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
|
@ -78,7 +64,7 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
|
|||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new BytesValues(bytes, termOrdToBytesOffset, ordinals.ordinals());
|
||||
return ordinals.ordinals(new ValuesHolder(bytes, termOrdToBytesOffset));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -86,82 +72,24 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
|
|||
return new ScriptDocValues.Strings(getBytesValues());
|
||||
}
|
||||
|
||||
@Override
|
||||
public TermsEnum getTermsEnum() {
|
||||
return new AtomicFieldDataWithOrdinalsTermsEnum(this);
|
||||
}
|
||||
private static class ValuesHolder implements Ordinals.ValuesHolder {
|
||||
|
||||
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
private final PagedBytes.Reader bytes;
|
||||
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
|
||||
|
||||
protected final PagedBytes.Reader bytes;
|
||||
protected final MonotonicAppendingLongBuffer termOrdToBytesOffset;
|
||||
protected final Ordinals.Docs ordinals;
|
||||
|
||||
BytesValues(PagedBytes.Reader bytes, MonotonicAppendingLongBuffer termOrdToBytesOffset, Ordinals.Docs ordinals) {
|
||||
super(ordinals);
|
||||
ValuesHolder(PagedBytes.Reader bytes, MonotonicAppendingLongBuffer termOrdToBytesOffset) {
|
||||
this.bytes = bytes;
|
||||
this.termOrdToBytesOffset = termOrdToBytesOffset;
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
// when we fill from the pages bytes, we just reference an existing buffer slice, its enough
|
||||
// to create a shallow copy of the bytes to be safe for "reads".
|
||||
return new BytesRef(scratch.bytes, scratch.offset, scratch.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public final Ordinals.Docs ordinals() {
|
||||
return this.ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final BytesRef getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
assert ord != BytesValues.WithOrdinals.MISSING_ORDINAL;
|
||||
bytes.fill(scratch, termOrdToBytesOffset.get(ord));
|
||||
return scratch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final BytesRef nextValue() {
|
||||
bytes.fill(scratch, termOrdToBytesOffset.get(ordinals.nextOrd()));
|
||||
return scratch;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final static class Empty extends PagedBytesAtomicFieldData {
|
||||
|
||||
Empty() {
|
||||
super(emptyBytes(), 0, new MonotonicAppendingLongBuffer(), EmptyOrdinals.INSTANCE);
|
||||
}
|
||||
|
||||
static PagedBytes.Reader emptyBytes() {
|
||||
PagedBytes bytes = new PagedBytes(1);
|
||||
bytes.copyUsingLengthPrefix(new BytesRef());
|
||||
return bytes.freeze(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScriptDocValues.Strings getScriptValues() {
|
||||
return ScriptDocValues.EMPTY_STRINGS;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -26,10 +26,7 @@ import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
|
|||
import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
|
||||
import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
|
@ -42,13 +39,13 @@ import java.io.IOException;
|
|||
|
||||
/**
|
||||
*/
|
||||
public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<PagedBytesAtomicFieldData> {
|
||||
public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> {
|
||||
|
||||
|
||||
public static class Builder implements IndexFieldData.Builder {
|
||||
|
||||
@Override
|
||||
public IndexFieldData<PagedBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
|
||||
public IndexFieldData<AtomicFieldData.WithOrdinals<ScriptDocValues.Strings>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper,
|
||||
IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService,
|
||||
GlobalOrdinalsBuilder globalOrdinalBuilder) {
|
||||
return new PagedBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache, breakerService, globalOrdinalBuilder);
|
||||
|
@ -62,15 +59,14 @@ public class PagedBytesIndexFieldData extends AbstractBytesIndexFieldData<PagedB
|
|||
}
|
||||
|
||||
@Override
|
||||
public PagedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
|
||||
public AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect(AtomicReaderContext context) throws Exception {
|
||||
AtomicReader reader = context.reader();
|
||||
|
||||
PagedBytesEstimator estimator = new PagedBytesEstimator(context, breakerService.getBreaker(), getFieldNames().fullName());
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
PagedBytesAtomicFieldData emptyData = PagedBytesAtomicFieldData.empty();
|
||||
estimator.adjustForNoTerms(emptyData.getMemorySizeInBytes());
|
||||
return emptyData;
|
||||
estimator.afterLoad(null, AtomicFieldData.WithOrdinals.EMPTY.getMemorySizeInBytes());
|
||||
return AtomicFieldData.WithOrdinals.EMPTY;
|
||||
}
|
||||
|
||||
final PagedBytes bytes = new PagedBytes(15);
|
||||
|
|
|
@ -31,16 +31,10 @@ import org.elasticsearch.index.fielddata.ScriptDocValues;
|
|||
public class ParentChildAtomicFieldData implements AtomicFieldData {
|
||||
|
||||
private final ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds;
|
||||
private final long numberUniqueValues;
|
||||
private final long memorySizeInBytes;
|
||||
|
||||
public ParentChildAtomicFieldData(ImmutableOpenMap<String, PagedBytesAtomicFieldData> typeToIds) {
|
||||
this.typeToIds = typeToIds;
|
||||
long numValues = 0;
|
||||
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
|
||||
numValues += cursor.value.getNumberUniqueValues();
|
||||
}
|
||||
this.numberUniqueValues = numValues;
|
||||
long size = 0;
|
||||
for (ObjectCursor<PagedBytesAtomicFieldData> cursor : typeToIds.values()) {
|
||||
size += cursor.value.getMemorySizeInBytes();
|
||||
|
@ -48,16 +42,6 @@ public class ParentChildAtomicFieldData implements AtomicFieldData {
|
|||
this.memorySizeInBytes = size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getNumberUniqueValues() {
|
||||
return numberUniqueValues;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return memorySizeInBytes;
|
||||
|
@ -72,6 +56,7 @@ public class ParentChildAtomicFieldData implements AtomicFieldData {
|
|||
}
|
||||
return new BytesValues(true) {
|
||||
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
private final BytesRef[] terms = new BytesRef[2];
|
||||
private int index;
|
||||
|
||||
|
|
|
@ -367,7 +367,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<ParentChil
|
|||
if(typeAfd != null) {
|
||||
return typeAfd;
|
||||
} else {
|
||||
return PagedBytesAtomicFieldData.empty();
|
||||
return AtomicFieldData.WithOrdinals.EMPTY;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.lucene.util.BytesRef;
|
|||
import org.elasticsearch.ElasticsearchIllegalStateException;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
@ -70,7 +69,7 @@ abstract class SortedSetDVAtomicFieldData {
|
|||
|
||||
public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getBytesValues() {
|
||||
final SortedSetDocValues values = getValuesNoException(reader, field);
|
||||
return new SortedSetValues(reader, field, values);
|
||||
return new SortedSetValues(values);
|
||||
}
|
||||
|
||||
public TermsEnum getTermsEnum() {
|
||||
|
@ -93,79 +92,23 @@ abstract class SortedSetDVAtomicFieldData {
|
|||
|
||||
static class SortedSetValues extends BytesValues.WithOrdinals {
|
||||
|
||||
protected final SortedSetDocValues values;
|
||||
|
||||
SortedSetValues(AtomicReader reader, String field, SortedSetDocValues values) {
|
||||
super(new SortedSetDocs(new SortedSetOrdinals(reader, field, values.getValueCount()), values));
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
assert ord != Ordinals.MISSING_ORDINAL;
|
||||
values.lookupOrd(ord, scratch);
|
||||
return scratch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef nextValue() {
|
||||
values.lookupOrd(ordinals.nextOrd(), scratch);
|
||||
return scratch;
|
||||
}
|
||||
}
|
||||
|
||||
static final class SortedSetOrdinals implements Ordinals {
|
||||
|
||||
// We don't store SortedSetDocValues as a member because Ordinals must be thread-safe
|
||||
private final AtomicReader reader;
|
||||
private final String field;
|
||||
private final long maxOrd;
|
||||
|
||||
public SortedSetOrdinals(AtomicReader reader, String field, long numOrds) {
|
||||
super();
|
||||
this.reader = reader;
|
||||
this.field = field;
|
||||
this.maxOrd = numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
// Ordinals can't be distinguished from the atomic field data instance
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return maxOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Docs ordinals() {
|
||||
final SortedSetDocValues values = getValuesNoException(reader, field);
|
||||
assert values.getValueCount() == maxOrd;
|
||||
return new SortedSetDocs(this, values);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class SortedSetDocs extends Ordinals.AbstractDocs {
|
||||
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
private final SortedSetDocValues values;
|
||||
private long[] ords;
|
||||
private int ordIndex = Integer.MAX_VALUE;
|
||||
private long currentOrdinal = -1;
|
||||
|
||||
SortedSetDocs(SortedSetOrdinals ordinals, SortedSetDocValues values) {
|
||||
super(ordinals);
|
||||
SortedSetValues(SortedSetDocValues values) {
|
||||
super(DocValues.unwrapSingleton(values) == null);
|
||||
this.values = values;
|
||||
ords = new long[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxOrd() {
|
||||
return values.getValueCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
values.setDocument(docId);
|
||||
|
@ -196,5 +139,11 @@ abstract class SortedSetDVAtomicFieldData {
|
|||
public long currentOrd() {
|
||||
return currentOrdinal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
values.lookupOrd(ord, scratch);
|
||||
return scratch;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.search.Queries;
|
|||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
||||
|
@ -107,8 +106,7 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
} else {
|
||||
AtomicFieldData.WithOrdinals afd = globalIfd.load(leaves.get(0));
|
||||
BytesValues.WithOrdinals globalValues = afd.getBytesValues();
|
||||
Ordinals.Docs globalOrdinals = globalValues.ordinals();
|
||||
maxOrd = globalOrdinals.getMaxOrd();
|
||||
maxOrd = globalValues.getMaxOrd();
|
||||
}
|
||||
|
||||
if (maxOrd == 0) {
|
||||
|
@ -242,8 +240,7 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
LongBitSet parentOrds = collector.parentOrds;
|
||||
BytesValues.WithOrdinals globalValues = globalIfd.load(context).getBytesValues();
|
||||
if (globalValues != null) {
|
||||
Ordinals.Docs globalOrdinals = globalValues.ordinals();
|
||||
DocIdSetIterator parentIdIterator = new ParentOrdIterator(innerIterator, parentOrds, globalOrdinals, this);
|
||||
DocIdSetIterator parentIdIterator = new ParentOrdIterator(innerIterator, parentOrds, globalValues, this);
|
||||
return ConstantScorer.create(parentIdIterator, this, queryWeight);
|
||||
}
|
||||
}
|
||||
|
@ -259,7 +256,6 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
private final ParentChildIndexFieldData.WithOrdinals indexFieldData;
|
||||
|
||||
private BytesValues.WithOrdinals values;
|
||||
private Ordinals.Docs globalOrdinals;
|
||||
|
||||
private ParentOrdCollector(ParentChildIndexFieldData.WithOrdinals indexFieldData, long maxOrd) {
|
||||
// TODO: look into reusing LongBitSet#bits array
|
||||
|
@ -269,9 +265,9 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
|
||||
@Override
|
||||
public void collect(int doc) throws IOException {
|
||||
if (globalOrdinals != null) {
|
||||
long globalOrdinal = globalOrdinals.getOrd(doc);
|
||||
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (values != null) {
|
||||
long globalOrdinal = values.getOrd(doc);
|
||||
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
parentOrds.set(globalOrdinal);
|
||||
}
|
||||
}
|
||||
|
@ -280,11 +276,6 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
@Override
|
||||
public void setNextReader(AtomicReaderContext context) throws IOException {
|
||||
values = indexFieldData.load(context).getBytesValues();
|
||||
if (values != null) {
|
||||
globalOrdinals = values.ordinals();
|
||||
} else {
|
||||
globalOrdinals = null;
|
||||
}
|
||||
}
|
||||
|
||||
long foundParents() {
|
||||
|
@ -296,10 +287,10 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
private final static class ParentOrdIterator extends FilteredDocIdSetIterator {
|
||||
|
||||
private final LongBitSet parentOrds;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final BytesValues.WithOrdinals ordinals;
|
||||
private final ParentWeight parentWeight;
|
||||
|
||||
private ParentOrdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, Ordinals.Docs ordinals, ParentWeight parentWeight) {
|
||||
private ParentOrdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, BytesValues.WithOrdinals ordinals, ParentWeight parentWeight) {
|
||||
super(innerIterator);
|
||||
this.parentOrds = parentOrds;
|
||||
this.ordinals = ordinals;
|
||||
|
@ -318,7 +309,7 @@ public class ChildrenConstantScoreQuery extends Query {
|
|||
}
|
||||
|
||||
long parentOrd = ordinals.getOrd(doc);
|
||||
if (parentOrd != Ordinals.MISSING_ORDINAL) {
|
||||
if (parentOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
boolean match = parentOrds.get(parentOrd);
|
||||
if (match) {
|
||||
parentWeight.remaining--;
|
||||
|
|
|
@ -37,7 +37,6 @@ import org.elasticsearch.common.util.IntArray;
|
|||
import org.elasticsearch.common.util.LongHash;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
import org.elasticsearch.search.internal.SearchContext.Lifetime;
|
||||
|
@ -289,25 +288,23 @@ public class ChildrenQuery extends Query {
|
|||
return null;
|
||||
}
|
||||
|
||||
Ordinals.Docs globalOrdinals = bytesValues.ordinals();
|
||||
|
||||
if (minChildren > 0 || maxChildren != 0 || scoreType == ScoreType.NONE) {
|
||||
switch (scoreType) {
|
||||
case NONE:
|
||||
DocIdSetIterator parentIdIterator = new CountParentOrdIterator(this, parents, collector, globalOrdinals,
|
||||
DocIdSetIterator parentIdIterator = new CountParentOrdIterator(this, parents, collector, bytesValues,
|
||||
minChildren, maxChildren);
|
||||
return ConstantScorer.create(parentIdIterator, this, queryWeight);
|
||||
case AVG:
|
||||
return new AvgParentCountScorer(this, parents, collector, globalOrdinals, minChildren, maxChildren);
|
||||
return new AvgParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
|
||||
default:
|
||||
return new ParentCountScorer(this, parents, collector, globalOrdinals, minChildren, maxChildren);
|
||||
return new ParentCountScorer(this, parents, collector, bytesValues, minChildren, maxChildren);
|
||||
}
|
||||
}
|
||||
switch (scoreType) {
|
||||
case AVG:
|
||||
return new AvgParentScorer(this, parents, collector, globalOrdinals);
|
||||
return new AvgParentScorer(this, parents, collector, bytesValues);
|
||||
default:
|
||||
return new ParentScorer(this, parents, collector, globalOrdinals);
|
||||
return new ParentScorer(this, parents, collector, bytesValues);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
|
@ -321,7 +318,6 @@ public class ChildrenQuery extends Query {
|
|||
protected final BigArrays bigArrays;
|
||||
protected final SearchContext searchContext;
|
||||
|
||||
protected Ordinals.Docs globalOrdinals;
|
||||
protected BytesValues.WithOrdinals values;
|
||||
protected Scorer scorer;
|
||||
|
||||
|
@ -334,9 +330,9 @@ public class ChildrenQuery extends Query {
|
|||
|
||||
@Override
|
||||
public final void collect(int doc) throws IOException {
|
||||
if (globalOrdinals != null) {
|
||||
final long globalOrdinal = globalOrdinals.getOrd(doc);
|
||||
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (values != null) {
|
||||
final long globalOrdinal = values.getOrd(doc);
|
||||
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
long parentIdx = parentIdxs.add(globalOrdinal);
|
||||
if (parentIdx >= 0) {
|
||||
newParent(parentIdx);
|
||||
|
@ -361,9 +357,6 @@ public class ChildrenQuery extends Query {
|
|||
@Override
|
||||
public void setNextReader(AtomicReaderContext context) throws IOException {
|
||||
values = globalIfd.load(context).getBytesValues();
|
||||
if (values != null) {
|
||||
globalOrdinals = values.ordinals();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -507,13 +500,13 @@ public class ChildrenQuery extends Query {
|
|||
final LongHash parentIds;
|
||||
final FloatArray scores;
|
||||
|
||||
final Ordinals.Docs globalOrdinals;
|
||||
final BytesValues.WithOrdinals globalOrdinals;
|
||||
final DocIdSetIterator parentsIterator;
|
||||
|
||||
int currentDocId = -1;
|
||||
float currentScore;
|
||||
|
||||
ParentScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals) {
|
||||
ParentScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals) {
|
||||
super(parentWeight);
|
||||
this.parentWeight = parentWeight;
|
||||
this.globalOrdinals = globalOrdinals;
|
||||
|
@ -557,7 +550,7 @@ public class ChildrenQuery extends Query {
|
|||
}
|
||||
|
||||
final long globalOrdinal = globalOrdinals.getOrd(currentDocId);
|
||||
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -583,7 +576,7 @@ public class ChildrenQuery extends Query {
|
|||
}
|
||||
|
||||
final long globalOrdinal = globalOrdinals.getOrd(currentDocId);
|
||||
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
return nextDoc();
|
||||
}
|
||||
|
||||
|
@ -609,7 +602,7 @@ public class ChildrenQuery extends Query {
|
|||
protected final int minChildren;
|
||||
protected final int maxChildren;
|
||||
|
||||
ParentCountScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals, int minChildren, int maxChildren) {
|
||||
ParentCountScorer(ParentWeight parentWeight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals, int minChildren, int maxChildren) {
|
||||
super(parentWeight, parentsIterator, (ParentScoreCollector) collector, globalOrdinals);
|
||||
this.minChildren = minChildren;
|
||||
this.maxChildren = maxChildren == 0 ? Integer.MAX_VALUE : maxChildren;
|
||||
|
@ -627,7 +620,7 @@ public class ChildrenQuery extends Query {
|
|||
|
||||
private static final class AvgParentScorer extends ParentCountScorer {
|
||||
|
||||
AvgParentScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals) {
|
||||
AvgParentScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals) {
|
||||
super(weight, parentsIterator, collector, globalOrdinals, 0, 0);
|
||||
}
|
||||
|
||||
|
@ -642,7 +635,7 @@ public class ChildrenQuery extends Query {
|
|||
|
||||
private static final class AvgParentCountScorer extends ParentCountScorer {
|
||||
|
||||
AvgParentCountScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, Ordinals.Docs globalOrdinals, int minChildren, int maxChildren) {
|
||||
AvgParentCountScorer(ParentWeight weight, DocIdSetIterator parentsIterator, ParentCollector collector, BytesValues.WithOrdinals globalOrdinals, int minChildren, int maxChildren) {
|
||||
super(weight, parentsIterator, collector, globalOrdinals, minChildren, maxChildren);
|
||||
}
|
||||
|
||||
|
@ -664,10 +657,10 @@ public class ChildrenQuery extends Query {
|
|||
protected final IntArray occurrences;
|
||||
private final int minChildren;
|
||||
private final int maxChildren;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final BytesValues.WithOrdinals ordinals;
|
||||
private final ParentWeight parentWeight;
|
||||
|
||||
private CountParentOrdIterator(ParentWeight parentWeight, DocIdSetIterator innerIterator, ParentCollector collector, Ordinals.Docs ordinals, int minChildren, int maxChildren) {
|
||||
private CountParentOrdIterator(ParentWeight parentWeight, DocIdSetIterator innerIterator, ParentCollector collector, BytesValues.WithOrdinals ordinals, int minChildren, int maxChildren) {
|
||||
super(innerIterator);
|
||||
this.parentIds = ((CountCollector) collector).parentIdxs;
|
||||
this.occurrences = ((CountCollector) collector).occurrences;
|
||||
|
@ -689,7 +682,7 @@ public class ChildrenQuery extends Query {
|
|||
}
|
||||
|
||||
final long parentOrd = ordinals.getOrd(doc);
|
||||
if (parentOrd != Ordinals.MISSING_ORDINAL) {
|
||||
if (parentOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
final long parentIdx = parentIds.find(parentOrd);
|
||||
if (parentIdx != -1) {
|
||||
parentWeight.remaining--;
|
||||
|
|
|
@ -31,7 +31,6 @@ import org.elasticsearch.common.lucene.search.Queries;
|
|||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -96,8 +95,7 @@ public class ParentConstantScoreQuery extends Query {
|
|||
} else {
|
||||
AtomicFieldData.WithOrdinals afd = globalIfd.load(leaves.get(0));
|
||||
BytesValues.WithOrdinals globalValues = afd.getBytesValues();
|
||||
Ordinals.Docs globalOrdinals = globalValues.ordinals();
|
||||
maxOrd = globalOrdinals.getMaxOrd();
|
||||
maxOrd = globalValues.getMaxOrd();
|
||||
}
|
||||
|
||||
if (maxOrd == 0) {
|
||||
|
@ -200,9 +198,8 @@ public class ParentConstantScoreQuery extends Query {
|
|||
if (globalValues != null) {
|
||||
DocIdSetIterator innerIterator = childrenDocIdSet.iterator();
|
||||
if (innerIterator != null) {
|
||||
Ordinals.Docs globalOrdinals = globalValues.ordinals();
|
||||
ChildrenDocIdIterator childrenDocIdIterator = new ChildrenDocIdIterator(
|
||||
innerIterator, parentOrds, globalOrdinals
|
||||
innerIterator, parentOrds, globalValues
|
||||
);
|
||||
return ConstantScorer.create(childrenDocIdIterator, this, queryWeight);
|
||||
}
|
||||
|
@ -215,9 +212,9 @@ public class ParentConstantScoreQuery extends Query {
|
|||
private final class ChildrenDocIdIterator extends FilteredDocIdSetIterator {
|
||||
|
||||
private final LongBitSet parentOrds;
|
||||
private final Ordinals.Docs globalOrdinals;
|
||||
private final BytesValues.WithOrdinals globalOrdinals;
|
||||
|
||||
ChildrenDocIdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, Ordinals.Docs globalOrdinals) {
|
||||
ChildrenDocIdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, BytesValues.WithOrdinals globalOrdinals) {
|
||||
super(innerIterator);
|
||||
this.parentOrds = parentOrds;
|
||||
this.globalOrdinals = globalOrdinals;
|
||||
|
@ -226,7 +223,7 @@ public class ParentConstantScoreQuery extends Query {
|
|||
@Override
|
||||
protected boolean match(int docId) {
|
||||
int globalOrd = (int) globalOrdinals.getOrd(docId);
|
||||
if (globalOrd != Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
return parentOrds.get(globalOrd);
|
||||
} else {
|
||||
return false;
|
||||
|
@ -240,7 +237,7 @@ public class ParentConstantScoreQuery extends Query {
|
|||
private final LongBitSet parentOrds;
|
||||
private final IndexFieldData.WithOrdinals globalIfd;
|
||||
|
||||
private Ordinals.Docs globalOrdinals;
|
||||
private BytesValues.WithOrdinals globalOrdinals;
|
||||
|
||||
ParentOrdsCollector(IndexFieldData.WithOrdinals globalIfd, long maxOrd) {
|
||||
this.parentOrds = new LongBitSet(maxOrd);
|
||||
|
@ -251,7 +248,7 @@ public class ParentConstantScoreQuery extends Query {
|
|||
// It can happen that for particular segment no document exist for an specific type. This prevents NPE
|
||||
if (globalOrdinals != null) {
|
||||
long globalOrd = globalOrdinals.getOrd(doc);
|
||||
if (globalOrd != Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrd != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
parentOrds.set(globalOrd);
|
||||
}
|
||||
}
|
||||
|
@ -259,10 +256,7 @@ public class ParentConstantScoreQuery extends Query {
|
|||
|
||||
@Override
|
||||
public void setNextReader(AtomicReaderContext readerContext) throws IOException {
|
||||
BytesValues.WithOrdinals values = globalIfd.load(readerContext).getBytesValues();
|
||||
if (values != null) {
|
||||
globalOrdinals = values.ordinals();
|
||||
}
|
||||
globalOrdinals = globalIfd.load(readerContext).getBytesValues();
|
||||
}
|
||||
|
||||
public long parentCount() {
|
||||
|
|
|
@ -36,7 +36,6 @@ import org.elasticsearch.common.util.FloatArray;
|
|||
import org.elasticsearch.common.util.LongHash;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
import org.elasticsearch.search.internal.SearchContext.Lifetime;
|
||||
|
@ -170,7 +169,6 @@ public class ParentQuery extends Query {
|
|||
|
||||
private Scorer scorer;
|
||||
private BytesValues.WithOrdinals values;
|
||||
private Ordinals.Docs globalOrdinals;
|
||||
|
||||
ParentOrdAndScoreCollector(SearchContext searchContext, IndexFieldData.WithOrdinals globalIfd) {
|
||||
this.bigArrays = searchContext.bigArrays();
|
||||
|
@ -182,9 +180,9 @@ public class ParentQuery extends Query {
|
|||
@Override
|
||||
public void collect(int doc) throws IOException {
|
||||
// It can happen that for particular segment no document exist for an specific type. This prevents NPE
|
||||
if (globalOrdinals != null) {
|
||||
long globalOrdinal = globalOrdinals.getOrd(doc);
|
||||
if (globalOrdinal != Ordinals.MISSING_ORDINAL) {
|
||||
if (values != null) {
|
||||
long globalOrdinal = values.getOrd(doc);
|
||||
if (globalOrdinal != BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
long parentIdx = parentIdxs.add(globalOrdinal);
|
||||
if (parentIdx >= 0) {
|
||||
scores = bigArrays.grow(scores, parentIdx + 1);
|
||||
|
@ -204,9 +202,6 @@ public class ParentQuery extends Query {
|
|||
@Override
|
||||
public void setNextReader(AtomicReaderContext context) throws IOException {
|
||||
values = globalIfd.load(context).getBytesValues();
|
||||
if (values != null) {
|
||||
globalOrdinals = values.ordinals();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -268,8 +263,7 @@ public class ParentQuery extends Query {
|
|||
return null;
|
||||
}
|
||||
|
||||
Ordinals.Docs ordinals = bytesValues.ordinals();
|
||||
return new ChildScorer(this, parentIdxs, scores, childrenDocSet.iterator(), ordinals);
|
||||
return new ChildScorer(this, parentIdxs, scores, childrenDocSet.iterator(), bytesValues);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -279,12 +273,12 @@ public class ParentQuery extends Query {
|
|||
private final LongHash parentIdxs;
|
||||
private final FloatArray scores;
|
||||
private final DocIdSetIterator childrenIterator;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final BytesValues.WithOrdinals ordinals;
|
||||
|
||||
private int currentChildDoc = -1;
|
||||
private float currentScore;
|
||||
|
||||
ChildScorer(Weight weight, LongHash parentIdxs, FloatArray scores, DocIdSetIterator childrenIterator, Ordinals.Docs ordinals) {
|
||||
ChildScorer(Weight weight, LongHash parentIdxs, FloatArray scores, DocIdSetIterator childrenIterator, BytesValues.WithOrdinals ordinals) {
|
||||
super(weight);
|
||||
this.parentIdxs = parentIdxs;
|
||||
this.scores = scores;
|
||||
|
@ -318,7 +312,7 @@ public class ParentQuery extends Query {
|
|||
}
|
||||
|
||||
int globalOrdinal = (int) ordinals.getOrd(currentChildDoc);
|
||||
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -338,7 +332,7 @@ public class ParentQuery extends Query {
|
|||
}
|
||||
|
||||
int globalOrdinal = (int) ordinals.getOrd(currentChildDoc);
|
||||
if (globalOrdinal == Ordinals.MISSING_ORDINAL) {
|
||||
if (globalOrdinal == BytesValues.WithOrdinals.MISSING_ORDINAL) {
|
||||
return nextDoc();
|
||||
}
|
||||
|
||||
|
|
|
@ -22,7 +22,7 @@ import org.apache.lucene.index.IndexReader;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.lease.Releasables;
|
||||
import org.elasticsearch.common.util.LongHash;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactories;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator;
|
||||
|
@ -61,14 +61,14 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
|
|||
@Override
|
||||
public SignificantStringTerms buildAggregation(long owningBucketOrdinal) {
|
||||
assert owningBucketOrdinal == 0;
|
||||
if (globalOrdinals == null) { // no context in this reader
|
||||
if (globalValues == null) { // no context in this reader
|
||||
return buildEmptyAggregation();
|
||||
}
|
||||
|
||||
final int size;
|
||||
if (bucketCountThresholds.getMinDocCount() == 0) {
|
||||
// if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
|
||||
size = (int) Math.min(globalOrdinals.getMaxOrd(), bucketCountThresholds.getShardSize());
|
||||
size = (int) Math.min(globalValues.getMaxOrd(), bucketCountThresholds.getShardSize());
|
||||
} else {
|
||||
size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
|
||||
}
|
||||
|
@ -77,7 +77,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
|
|||
|
||||
BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size);
|
||||
SignificantStringTerms.Bucket spare = null;
|
||||
for (long globalTermOrd = Ordinals.MIN_ORDINAL; globalTermOrd < globalOrdinals.getMaxOrd(); ++globalTermOrd) {
|
||||
for (long globalTermOrd = BytesValues.WithOrdinals.MIN_ORDINAL; globalTermOrd < globalValues.getMaxOrd(); ++globalTermOrd) {
|
||||
if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
|
||||
continue;
|
||||
}
|
||||
|
@ -143,9 +143,9 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
|
|||
@Override
|
||||
public void collect(int doc, long owningBucketOrdinal) throws IOException {
|
||||
numCollectedDocs++;
|
||||
final int numOrds = globalOrdinals.setDocument(doc);
|
||||
final int numOrds = globalValues.setDocument(doc);
|
||||
for (int i = 0; i < numOrds; i++) {
|
||||
final long globalOrd = globalOrdinals.nextOrd();
|
||||
final long globalOrd = globalValues.nextOrd();
|
||||
long bucketOrd = bucketOrds.add(globalOrd);
|
||||
if (bucketOrd < 0) {
|
||||
bucketOrd = -1 - bucketOrd;
|
||||
|
|
|
@ -30,8 +30,7 @@ import org.elasticsearch.common.text.Text;
|
|||
import org.elasticsearch.common.util.IntArray;
|
||||
import org.elasticsearch.common.util.LongHash;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.GlobalOrdinalMapping;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactories;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
|
@ -54,13 +53,12 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
protected final IncludeExclude includeExclude;
|
||||
|
||||
protected BytesValues.WithOrdinals globalValues;
|
||||
protected Ordinals.Docs globalOrdinals;
|
||||
|
||||
// TODO: cache the acceptedGlobalOrdinals per aggregation definition.
|
||||
// TODO: cache the acceptedglobalValues per aggregation definition.
|
||||
// We can't cache this yet in ValuesSource, since ValuesSource is reused per field for aggs during the execution.
|
||||
// If aggs with same field, but different include/exclude are defined, then the last defined one will override the
|
||||
// first defined one.
|
||||
// So currently for each instance of this aggregator the acceptedGlobalOrdinals will be computed, this is unnecessary
|
||||
// So currently for each instance of this aggregator the acceptedglobalValues will be computed, this is unnecessary
|
||||
// especially if this agg is on a second layer or deeper.
|
||||
protected LongBitSet acceptedGlobalOrdinals;
|
||||
|
||||
|
@ -84,20 +82,19 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
@Override
|
||||
public void setNextReader(AtomicReaderContext reader) {
|
||||
globalValues = valuesSource.globalBytesValues();
|
||||
globalOrdinals = globalValues.ordinals();
|
||||
if (acceptedGlobalOrdinals != null) {
|
||||
globalOrdinals = new FilteredOrdinals(globalOrdinals, acceptedGlobalOrdinals);
|
||||
globalValues = new FilteredOrdinals(globalValues, acceptedGlobalOrdinals);
|
||||
} else if (includeExclude != null) {
|
||||
acceptedGlobalOrdinals = includeExclude.acceptedGlobalOrdinals(globalOrdinals, valuesSource);
|
||||
globalOrdinals = new FilteredOrdinals(globalOrdinals, acceptedGlobalOrdinals);
|
||||
acceptedGlobalOrdinals = includeExclude.acceptedGlobalOrdinals(globalValues, valuesSource);
|
||||
globalValues = new FilteredOrdinals(globalValues, acceptedGlobalOrdinals);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void collect(int doc, long owningBucketOrdinal) throws IOException {
|
||||
final int numOrds = globalOrdinals.setDocument(doc);
|
||||
final int numOrds = globalValues.setDocument(doc);
|
||||
for (int i = 0; i < numOrds; i++) {
|
||||
final long globalOrd = globalOrdinals.nextOrd();
|
||||
final long globalOrd = globalValues.nextOrd();
|
||||
collectExistingBucket(doc, globalOrd);
|
||||
}
|
||||
}
|
||||
|
@ -113,20 +110,20 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
|
||||
@Override
|
||||
public InternalAggregation buildAggregation(long owningBucketOrdinal) {
|
||||
if (globalOrdinals == null) { // no context in this reader
|
||||
if (globalValues == null) { // no context in this reader
|
||||
return buildEmptyAggregation();
|
||||
}
|
||||
|
||||
final int size;
|
||||
if (bucketCountThresholds.getMinDocCount() == 0) {
|
||||
// if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns
|
||||
size = (int) Math.min(globalOrdinals.getMaxOrd(), bucketCountThresholds.getShardSize());
|
||||
size = (int) Math.min(globalValues.getMaxOrd(), bucketCountThresholds.getShardSize());
|
||||
} else {
|
||||
size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
|
||||
}
|
||||
BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(this));
|
||||
OrdBucket spare = new OrdBucket(-1, 0, null);
|
||||
for (long globalTermOrd = Ordinals.MIN_ORDINAL; globalTermOrd < globalOrdinals.getMaxOrd(); ++globalTermOrd) {
|
||||
for (long globalTermOrd = BytesValues.WithOrdinals.MIN_ORDINAL; globalTermOrd < globalValues.getMaxOrd(); ++globalTermOrd) {
|
||||
if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
|
||||
continue;
|
||||
}
|
||||
|
@ -228,9 +225,9 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
|
||||
@Override
|
||||
public void collect(int doc, long owningBucketOrdinal) throws IOException {
|
||||
final int numOrds = globalOrdinals.setDocument(doc);
|
||||
final int numOrds = globalValues.setDocument(doc);
|
||||
for (int i = 0; i < numOrds; i++) {
|
||||
final long globalOrd = globalOrdinals.nextOrd();
|
||||
final long globalOrd = globalValues.nextOrd();
|
||||
long bucketOrd = bucketOrds.add(globalOrd);
|
||||
if (bucketOrd < 0) {
|
||||
bucketOrd = -1 - bucketOrd;
|
||||
|
@ -262,7 +259,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
|
||||
private final IntArray segmentDocCounts;
|
||||
|
||||
private Ordinals.Docs segmentOrdinals;
|
||||
private BytesValues.WithOrdinals segmentOrdinals;
|
||||
private IntArray current;
|
||||
|
||||
public LowCardinality(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals.FieldData valuesSource, long estimatedBucketCount,
|
||||
|
@ -282,16 +279,14 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
|
||||
@Override
|
||||
public void setNextReader(AtomicReaderContext reader) {
|
||||
if (segmentOrdinals != null && segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
|
||||
if (segmentOrdinals != null && segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
|
||||
mapSegmentCountsToGlobalCounts();
|
||||
}
|
||||
|
||||
globalValues = valuesSource.globalBytesValues();
|
||||
globalOrdinals = globalValues.ordinals();
|
||||
|
||||
BytesValues.WithOrdinals bytesValues = valuesSource.bytesValues();
|
||||
segmentOrdinals = bytesValues.ordinals();
|
||||
if (segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
|
||||
segmentOrdinals = valuesSource.bytesValues();
|
||||
if (segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
|
||||
current = segmentDocCounts;
|
||||
} else {
|
||||
current = getDocCounts();
|
||||
|
@ -300,7 +295,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
|
||||
@Override
|
||||
protected void doPostCollection() {
|
||||
if (segmentOrdinals.getMaxOrd() != globalOrdinals.getMaxOrd()) {
|
||||
if (segmentOrdinals.getMaxOrd() != globalValues.getMaxOrd()) {
|
||||
mapSegmentCountsToGlobalCounts();
|
||||
}
|
||||
}
|
||||
|
@ -313,7 +308,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
private void mapSegmentCountsToGlobalCounts() {
|
||||
// There is no public method in Ordinals.Docs that allows for this mapping...
|
||||
// This is the cleanest way I can think of so far
|
||||
GlobalOrdinalMapping mapping = (GlobalOrdinalMapping) globalOrdinals;
|
||||
GlobalOrdinalMapping mapping = (GlobalOrdinalMapping) globalValues;
|
||||
for (int i = 0; i < segmentDocCounts.size(); i++) {
|
||||
final int inc = segmentDocCounts.set(i, 0);
|
||||
if (inc == 0) {
|
||||
|
@ -329,16 +324,17 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
}
|
||||
}
|
||||
|
||||
private static final class FilteredOrdinals implements Ordinals.Docs {
|
||||
private static final class FilteredOrdinals extends BytesValues.WithOrdinals {
|
||||
|
||||
private final Ordinals.Docs inner;
|
||||
private final BytesValues.WithOrdinals inner;
|
||||
private final LongBitSet accepted;
|
||||
|
||||
private long currentOrd;
|
||||
private long[] buffer = new long[0];
|
||||
private int bufferSlot;
|
||||
|
||||
private FilteredOrdinals(Ordinals.Docs inner, LongBitSet accepted) {
|
||||
private FilteredOrdinals(BytesValues.WithOrdinals inner, LongBitSet accepted) {
|
||||
super(inner.isMultiValued());
|
||||
this.inner = inner;
|
||||
this.accepted = accepted;
|
||||
}
|
||||
|
@ -348,18 +344,13 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
return inner.getMaxOrd();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return inner.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getOrd(int docId) {
|
||||
long ord = inner.getOrd(docId);
|
||||
if (accepted.get(ord)) {
|
||||
return currentOrd = ord;
|
||||
} else {
|
||||
return currentOrd = Ordinals.MISSING_ORDINAL;
|
||||
return currentOrd = MISSING_ORDINAL;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -389,5 +380,15 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
|
|||
public long currentOrd() {
|
||||
return currentOrd;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(long ord) {
|
||||
return inner.getValueByOrd(ord);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef copyShared() {
|
||||
return inner.copyShared();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -28,7 +28,6 @@ import org.elasticsearch.common.collect.Iterators2;
|
|||
import org.elasticsearch.common.lease.Releasables;
|
||||
import org.elasticsearch.common.util.BytesRefHash;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.AggregatorFactories;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
|
@ -92,15 +91,14 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
|
|||
|
||||
/** Returns an iterator over the field data terms. */
|
||||
private static Iterator<BytesRef> terms(final BytesValues.WithOrdinals bytesValues, boolean reverse) {
|
||||
final Ordinals.Docs ordinals = bytesValues.ordinals();
|
||||
if (reverse) {
|
||||
return new UnmodifiableIterator<BytesRef>() {
|
||||
|
||||
long i = ordinals.getMaxOrd() - 1;
|
||||
long i = bytesValues.getMaxOrd() - 1;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return i >= Ordinals.MIN_ORDINAL;
|
||||
return i >= BytesValues.WithOrdinals.MIN_ORDINAL;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -113,11 +111,11 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
|
|||
} else {
|
||||
return new UnmodifiableIterator<BytesRef>() {
|
||||
|
||||
long i = Ordinals.MIN_ORDINAL;
|
||||
long i = BytesValues.WithOrdinals.MIN_ORDINAL;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return i < ordinals.getMaxOrd();
|
||||
return i < bytesValues.getMaxOrd();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -19,11 +19,14 @@
|
|||
package org.elasticsearch.search.aggregations.bucket.terms.support;
|
||||
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.util.*;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRef;
|
||||
import org.apache.lucene.util.LongBitSet;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.elasticsearch.ExceptionsHelper;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
@ -75,7 +78,7 @@ public class IncludeExclude {
|
|||
/**
|
||||
* Computes which global ordinals are accepted by this IncludeExclude instance.
|
||||
*/
|
||||
public LongBitSet acceptedGlobalOrdinals(Ordinals.Docs globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) {
|
||||
public LongBitSet acceptedGlobalOrdinals(BytesValues.WithOrdinals globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) {
|
||||
TermsEnum globalTermsEnum = valueSource.getGlobalTermsEnum();
|
||||
LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getMaxOrd());
|
||||
try {
|
||||
|
|
|
@ -34,7 +34,6 @@ import org.elasticsearch.common.util.ObjectArray;
|
|||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.LongValues;
|
||||
import org.elasticsearch.index.fielddata.MurmurHash3Values;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.InternalAggregation;
|
||||
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
|
||||
|
@ -92,7 +91,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
|
|||
final BytesValues bytesValues = valuesSource.bytesValues();
|
||||
if (bytesValues instanceof BytesValues.WithOrdinals) {
|
||||
BytesValues.WithOrdinals values = (BytesValues.WithOrdinals) bytesValues;
|
||||
final long maxOrd = values.ordinals().getMaxOrd();
|
||||
final long maxOrd = values.getMaxOrd();
|
||||
if (maxOrd == 0) {
|
||||
return new EmptyCollector();
|
||||
}
|
||||
|
@ -231,15 +230,13 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
|
|||
|
||||
private final BigArrays bigArrays;
|
||||
private final BytesValues.WithOrdinals values;
|
||||
private final Ordinals.Docs ordinals;
|
||||
private final int maxOrd;
|
||||
private final HyperLogLogPlusPlus counts;
|
||||
private ObjectArray<FixedBitSet> visitedOrds;
|
||||
|
||||
OrdinalsCollector(HyperLogLogPlusPlus counts, BytesValues.WithOrdinals values, BigArrays bigArrays) {
|
||||
ordinals = values.ordinals();
|
||||
Preconditions.checkArgument(ordinals.getMaxOrd() <= Integer.MAX_VALUE);
|
||||
maxOrd = (int) ordinals.getMaxOrd();
|
||||
Preconditions.checkArgument(values.getMaxOrd() <= Integer.MAX_VALUE);
|
||||
maxOrd = (int) values.getMaxOrd();
|
||||
this.bigArrays = bigArrays;
|
||||
this.counts = counts;
|
||||
this.values = values;
|
||||
|
@ -254,9 +251,9 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
|
|||
bits = new FixedBitSet(maxOrd);
|
||||
visitedOrds.set(bucketOrd, bits);
|
||||
}
|
||||
final int valueCount = ordinals.setDocument(doc);
|
||||
final int valueCount = values.setDocument(doc);
|
||||
for (int i = 0; i < valueCount; ++i) {
|
||||
bits.set((int) ordinals.nextOrd());
|
||||
bits.set((int) values.nextOrd());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.TopReaderContextAware;
|
|||
import org.elasticsearch.common.util.CollectionUtils;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData.Order;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.script.SearchScript;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.SortedAndUnique.SortedUniqueBytesValues;
|
||||
import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues;
|
||||
|
@ -91,8 +90,25 @@ public abstract class ValuesSource {
|
|||
metaData.uniqueness = Uniqueness.UNIQUE;
|
||||
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
|
||||
AtomicFieldData fieldData = indexFieldData.load(readerContext);
|
||||
metaData.multiValued |= fieldData.isMultiValued();
|
||||
metaData.maxAtomicUniqueValuesCount = Math.max(metaData.maxAtomicUniqueValuesCount, fieldData.getNumberUniqueValues());
|
||||
if (fieldData instanceof AtomicFieldData.WithOrdinals<?>) {
|
||||
AtomicFieldData.WithOrdinals<?> fd = (AtomicFieldData.WithOrdinals<?>) fieldData;
|
||||
BytesValues.WithOrdinals values = fd.getBytesValues();
|
||||
metaData.multiValued |= values.isMultiValued();
|
||||
metaData.maxAtomicUniqueValuesCount = Math.max(metaData.maxAtomicUniqueValuesCount, values.getMaxOrd());
|
||||
} else if (fieldData instanceof AtomicNumericFieldData) {
|
||||
AtomicNumericFieldData fd = (AtomicNumericFieldData) fieldData;
|
||||
DoubleValues values = fd.getDoubleValues();
|
||||
metaData.multiValued |= values.isMultiValued();
|
||||
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
|
||||
} else if (fieldData instanceof AtomicGeoPointFieldData<?>) {
|
||||
AtomicGeoPointFieldData<?> fd = (AtomicGeoPointFieldData<?>) fieldData;
|
||||
GeoPointValues values = fd.getGeoPointValues();
|
||||
metaData.multiValued |= values.isMultiValued();
|
||||
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
|
||||
} else {
|
||||
metaData.multiValued = true;
|
||||
metaData.maxAtomicUniqueValuesCount = Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
return metaData;
|
||||
}
|
||||
|
@ -243,8 +259,7 @@ public abstract class ValuesSource {
|
|||
IndexFieldData.WithOrdinals<?> globalFieldData = indexFieldData.loadGlobal(indexReader);
|
||||
AtomicFieldData.WithOrdinals afd = globalFieldData.load(atomicReaderContext);
|
||||
BytesValues.WithOrdinals values = afd.getBytesValues();
|
||||
Ordinals.Docs ordinals = values.ordinals();
|
||||
return maxOrd = ordinals.getMaxOrd();
|
||||
return maxOrd = values.getMaxOrd();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -349,6 +364,7 @@ public abstract class ValuesSource {
|
|||
}
|
||||
|
||||
static class SortedUniqueBytesValues extends BytesValues {
|
||||
final BytesRef scratch = new BytesRef();
|
||||
final BytesValues delegate;
|
||||
int[] indices = new int[1]; // at least one
|
||||
final BytesRefArray bytes;
|
||||
|
@ -758,6 +774,7 @@ public abstract class ValuesSource {
|
|||
|
||||
static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues {
|
||||
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
private final ValuesSource source;
|
||||
private final SearchScript script;
|
||||
|
||||
|
|
|
@ -33,6 +33,7 @@ import java.util.Iterator;
|
|||
*/
|
||||
public class ScriptBytesValues extends BytesValues implements ScriptValues {
|
||||
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
final SearchScript script;
|
||||
|
||||
private Iterator<?> iter;
|
||||
|
@ -50,7 +51,6 @@ public class ScriptBytesValues extends BytesValues implements ScriptValues {
|
|||
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
this.docId = docId;
|
||||
script.setNextDocId(docId);
|
||||
value = script.run();
|
||||
|
||||
|
|
|
@ -30,7 +30,6 @@ import org.elasticsearch.common.collect.BoundedTreeSet;
|
|||
import org.elasticsearch.common.recycler.Recycler;
|
||||
import org.elasticsearch.index.fielddata.DoubleValues;
|
||||
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.script.SearchScript;
|
||||
import org.elasticsearch.search.facet.DoubleFacetAggregatorBase;
|
||||
import org.elasticsearch.search.facet.FacetExecutor;
|
||||
|
@ -74,19 +73,11 @@ public class TermsDoubleFacetExecutor extends FacetExecutor {
|
|||
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
|
||||
int maxDoc = readerContext.reader().maxDoc();
|
||||
DoubleValues values = indexFieldData.load(readerContext).getDoubleValues();
|
||||
if (values instanceof DoubleValues.WithOrdinals) {
|
||||
DoubleValues.WithOrdinals valuesWithOrds = (DoubleValues.WithOrdinals) values;
|
||||
Ordinals.Docs ordinals = valuesWithOrds.ordinals();
|
||||
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
|
||||
facets.v().putIfAbsent(valuesWithOrds.getValueByOrd(ord), 0);
|
||||
}
|
||||
} else {
|
||||
for (int docId = 0; docId < maxDoc; docId++) {
|
||||
int numValues = values.setDocument(docId);
|
||||
DoubleIntOpenHashMap map = facets.v();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
map.putIfAbsent(values.nextValue(), 0);
|
||||
}
|
||||
for (int docId = 0; docId < maxDoc; docId++) {
|
||||
int numValues = values.setDocument(docId);
|
||||
DoubleIntOpenHashMap map = facets.v();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
map.putIfAbsent(values.nextValue(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -30,7 +30,6 @@ import org.elasticsearch.common.collect.BoundedTreeSet;
|
|||
import org.elasticsearch.common.recycler.Recycler;
|
||||
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
|
||||
import org.elasticsearch.index.fielddata.LongValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.script.SearchScript;
|
||||
import org.elasticsearch.search.facet.FacetExecutor;
|
||||
import org.elasticsearch.search.facet.InternalFacet;
|
||||
|
@ -73,19 +72,11 @@ public class TermsLongFacetExecutor extends FacetExecutor {
|
|||
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
|
||||
int maxDoc = readerContext.reader().maxDoc();
|
||||
LongValues values = indexFieldData.load(readerContext).getLongValues();
|
||||
if (values instanceof LongValues.WithOrdinals) {
|
||||
LongValues.WithOrdinals valuesWithOrds = (LongValues.WithOrdinals) values;
|
||||
Ordinals.Docs ordinals = valuesWithOrds.ordinals();
|
||||
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
|
||||
facets.v().putIfAbsent(valuesWithOrds.getValueByOrd(ord), 0);
|
||||
}
|
||||
} else {
|
||||
for (int docId = 0; docId < maxDoc; docId++) {
|
||||
final int numValues = values.setDocument(docId);
|
||||
final LongIntOpenHashMap v = facets.v();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
v.putIfAbsent(values.nextValue(), 0);
|
||||
}
|
||||
for (int docId = 0; docId < maxDoc; docId++) {
|
||||
final int numValues = values.setDocument(docId);
|
||||
final LongIntOpenHashMap v = facets.v();
|
||||
for (int i = 0; i < numValues; i++) {
|
||||
v.putIfAbsent(values.nextValue(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -25,7 +25,6 @@ import org.apache.lucene.search.Scorer;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.script.SearchScript;
|
||||
import org.elasticsearch.search.facet.FacetExecutor;
|
||||
import org.elasticsearch.search.facet.InternalFacet;
|
||||
|
@ -131,9 +130,8 @@ public class TermsStringFacetExecutor extends FacetExecutor {
|
|||
int maxDoc = readerContext.reader().maxDoc();
|
||||
if (indexFieldData instanceof IndexFieldData.WithOrdinals) {
|
||||
BytesValues.WithOrdinals values = ((IndexFieldData.WithOrdinals) indexFieldData).load(readerContext).getBytesValues();
|
||||
Ordinals.Docs ordinals = values.ordinals();
|
||||
// 0 = docs with no value for field, so start from 1 instead
|
||||
for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
|
||||
for (long ord = BytesValues.WithOrdinals.MIN_ORDINAL; ord < values.getMaxOrd(); ord++) {
|
||||
BytesRef value = values.getValueByOrd(ord);
|
||||
aggregator.addValue(value, value.hashCode(), values);
|
||||
}
|
||||
|
|
|
@ -33,7 +33,6 @@ import org.elasticsearch.common.util.BigArrays;
|
|||
import org.elasticsearch.common.util.IntArray;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.search.facet.FacetExecutor;
|
||||
import org.elasticsearch.search.facet.InternalFacet;
|
||||
import org.elasticsearch.search.facet.terms.TermsFacet;
|
||||
|
@ -200,14 +199,13 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
|
|||
private long total;
|
||||
private BytesValues.WithOrdinals values;
|
||||
private ReaderAggregator current;
|
||||
private Ordinals.Docs ordinals;
|
||||
|
||||
@Override
|
||||
public void setNextReader(AtomicReaderContext context) throws IOException {
|
||||
if (current != null) {
|
||||
missing += current.missing;
|
||||
total += current.total;
|
||||
if (current.values.ordinals().getMaxOrd() > Ordinals.MIN_ORDINAL) {
|
||||
if (current.values.getMaxOrd() > BytesValues.WithOrdinals.MIN_ORDINAL) {
|
||||
aggregators.add(current);
|
||||
} else {
|
||||
Releasables.close(current);
|
||||
|
@ -215,15 +213,14 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
|
|||
}
|
||||
values = indexFieldData.load(context).getBytesValues();
|
||||
current = new ReaderAggregator(values, ordinalsCacheAbove, cacheRecycler);
|
||||
ordinals = values.ordinals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void collect(int doc) throws IOException {
|
||||
final int length = ordinals.setDocument(doc);
|
||||
final int length = values.setDocument(doc);
|
||||
int missing = 1;
|
||||
for (int i = 0; i < length; i++) {
|
||||
current.onOrdinal(doc, ordinals.nextOrd());
|
||||
current.onOrdinal(doc, values.nextOrd());
|
||||
missing = 0;
|
||||
}
|
||||
current.incrementMissing(missing);
|
||||
|
@ -235,7 +232,7 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
|
|||
missing += current.missing;
|
||||
total += current.total;
|
||||
// if we have values for this one, add it
|
||||
if (current.values.ordinals().getMaxOrd() > Ordinals.MIN_ORDINAL) {
|
||||
if (current.values.getMaxOrd() > BytesValues.WithOrdinals.MIN_ORDINAL) {
|
||||
aggregators.add(current);
|
||||
} else {
|
||||
Releasables.close(current);
|
||||
|
@ -254,14 +251,14 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
|
|||
final BytesValues.WithOrdinals values;
|
||||
final IntArray counts;
|
||||
int missing = 0;
|
||||
long position = Ordinals.MIN_ORDINAL - 1;
|
||||
long position = BytesValues.WithOrdinals.MIN_ORDINAL - 1;
|
||||
BytesRef current;
|
||||
int total;
|
||||
|
||||
|
||||
public ReaderAggregator(BytesValues.WithOrdinals values, int ordinalsCacheLimit, CacheRecycler cacheRecycler) {
|
||||
this.values = values;
|
||||
this.maxOrd = values.ordinals().getMaxOrd();
|
||||
this.maxOrd = values.getMaxOrd();
|
||||
this.counts = bigArrays.newIntArray(maxOrd);
|
||||
}
|
||||
|
||||
|
|
|
@ -34,15 +34,12 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.SizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.discovery.Discovery;
|
||||
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.indices.IndexAlreadyExistsException;
|
||||
import org.elasticsearch.node.internal.InternalNode;
|
||||
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
||||
import org.elasticsearch.transport.TransportModule;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
|
||||
|
@ -174,20 +171,14 @@ public class GlobalOrdinalsBenchmark {
|
|||
System.out.println("--> Number of docs in index: " + COUNT);
|
||||
|
||||
List<StatsResult> stats = new ArrayList<>();
|
||||
int[] thresholds = new int[]{2048};
|
||||
for (int threshold : thresholds) {
|
||||
updateThresholdInMapping(threshold);
|
||||
System.out.println("--> Threshold: " + threshold);
|
||||
|
||||
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
|
||||
String fieldName = "field_" + fieldSuffix;
|
||||
String name = "global_ordinals-" + fieldName;
|
||||
if (USE_DOC_VALUES) {
|
||||
fieldName = fieldName + ".doc_values";
|
||||
name = name + "_doc_values"; // can't have . in agg name
|
||||
}
|
||||
stats.add(terms(name, fieldName, "global_ordinals_low_cardinality"));
|
||||
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
|
||||
String fieldName = "field_" + fieldSuffix;
|
||||
String name = "global_ordinals-" + fieldName;
|
||||
if (USE_DOC_VALUES) {
|
||||
fieldName = fieldName + ".doc_values";
|
||||
name = name + "_doc_values"; // can't have . in agg name
|
||||
}
|
||||
stats.add(terms(name, fieldName, "global_ordinals_low_cardinality"));
|
||||
}
|
||||
|
||||
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
|
||||
|
@ -211,22 +202,6 @@ public class GlobalOrdinalsBenchmark {
|
|||
node.close();
|
||||
}
|
||||
|
||||
private static void updateThresholdInMapping(int threshold) throws IOException {
|
||||
XContentBuilder builder = jsonBuilder().startObject().startObject(TYPE_NAME).startObject("properties");
|
||||
for (int fieldSuffix = FIELD_START; fieldSuffix <= FIELD_LIMIT; fieldSuffix <<= 1) {
|
||||
builder.startObject("field_" + fieldSuffix)
|
||||
.field("type", "string")
|
||||
.field("index", "not_analyzed")
|
||||
.startObject("fielddata")
|
||||
.field(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_KEY, threshold)
|
||||
.endObject()
|
||||
.endObject();
|
||||
}
|
||||
builder = builder.endObject().endObject().endObject();
|
||||
|
||||
client.admin().indices().preparePutMapping(INDEX_NAME).setType(TYPE_NAME).setSource(builder).get();
|
||||
}
|
||||
|
||||
private static StatsResult terms(String name, String field, String executionHint) {
|
||||
long totalQueryTime;// LM VALUE
|
||||
|
||||
|
|
|
@ -43,7 +43,6 @@ import org.elasticsearch.common.settings.ImmutableSettings;
|
|||
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
|
||||
import org.elasticsearch.search.MultiValueMode;
|
||||
import org.junit.Test;
|
||||
|
@ -427,7 +426,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
|
|||
public void testGlobalOrdinals() throws Exception {
|
||||
fillExtendedMvSet();
|
||||
refreshReader();
|
||||
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_ordinals", "fixed"));
|
||||
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_values", "fixed"));
|
||||
IndexFieldData.WithOrdinals ifd = getForField(fieldDataType, "value");
|
||||
IndexFieldData.WithOrdinals globalOrdinals = ifd.loadGlobal(topLevelReader);
|
||||
assertThat(topLevelReader.leaves().size(), equalTo(3));
|
||||
|
@ -436,68 +435,65 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
|
|||
assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class));
|
||||
AtomicFieldData.WithOrdinals afd = globalOrdinals.load(topLevelReader.leaves().get(0));
|
||||
BytesValues.WithOrdinals values = afd.getBytesValues();
|
||||
Ordinals.Docs ordinals = afd.getBytesValues().ordinals();
|
||||
assertThat(ordinals.setDocument(0), equalTo(2));
|
||||
long ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(0), equalTo(2));
|
||||
long ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(3l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("02"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(5l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("04"));
|
||||
assertThat(ordinals.setDocument(1), equalTo(0));
|
||||
assertThat(ordinals.setDocument(2), equalTo(1));
|
||||
ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(1), equalTo(0));
|
||||
assertThat(values.setDocument(2), equalTo(1));
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(4l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("03"));
|
||||
|
||||
// Second segment
|
||||
afd = globalOrdinals.load(topLevelReader.leaves().get(1));
|
||||
values = afd.getBytesValues();
|
||||
ordinals = afd.getBytesValues().ordinals();
|
||||
assertThat(ordinals.setDocument(0), equalTo(3));
|
||||
ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(0), equalTo(3));
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(5l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("04"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(6l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("05"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(7l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("06"));
|
||||
assertThat(ordinals.setDocument(1), equalTo(3));
|
||||
ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(1), equalTo(3));
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(7l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("06"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(8l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("07"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(9l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("08"));
|
||||
assertThat(ordinals.setDocument(2), equalTo(0));
|
||||
assertThat(ordinals.setDocument(3), equalTo(3));
|
||||
ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(2), equalTo(0));
|
||||
assertThat(values.setDocument(3), equalTo(3));
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(9l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("08"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(10l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("09"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(11l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("10"));
|
||||
|
||||
// Third segment
|
||||
afd = globalOrdinals.load(topLevelReader.leaves().get(2));
|
||||
values = afd.getBytesValues();
|
||||
ordinals = afd.getBytesValues().ordinals();
|
||||
assertThat(ordinals.setDocument(0), equalTo(3));
|
||||
ord = ordinals.nextOrd();
|
||||
assertThat(values.setDocument(0), equalTo(3));
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(0l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!08"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(1l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!09"));
|
||||
ord = ordinals.nextOrd();
|
||||
ord = values.nextOrd();
|
||||
assertThat(ord, equalTo(2l));
|
||||
assertThat(values.getValueByOrd(ord).utf8ToString(), equalTo("!10"));
|
||||
}
|
||||
|
@ -542,7 +538,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
|
|||
public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception {
|
||||
fillExtendedMvSet();
|
||||
refreshReader();
|
||||
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_ordinals", "fixed"));
|
||||
FieldDataType fieldDataType = new FieldDataType("string", ImmutableSettings.builder().put("global_values", "fixed"));
|
||||
IndexFieldData.WithOrdinals ifd = getForField(fieldDataType, "value");
|
||||
IndexFieldData.WithOrdinals globalOrdinals = ifd.loadGlobal(topLevelReader);
|
||||
assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));
|
||||
|
|
|
@ -38,8 +38,6 @@ import org.elasticsearch.common.unit.DistanceUnit;
|
|||
import org.elasticsearch.common.unit.DistanceUnit.Distance;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||
import org.elasticsearch.index.mapper.MapperTestUtils;
|
||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||
|
@ -404,10 +402,8 @@ public class DuelFieldDataTests extends AbstractFieldDataTests {
|
|||
BytesValues.WithOrdinals left = fieldData.load(readerContext).getBytesValues();
|
||||
fieldData.clear();
|
||||
BytesValues.WithOrdinals right = fieldData.loadGlobal(topLevelReader).load(topLevelReader.leaves().get(0)).getBytesValues();
|
||||
Docs leftOrds = left.ordinals();
|
||||
Docs rightOrds = right.ordinals();
|
||||
assertEquals(leftOrds.getMaxOrd(), rightOrds.getMaxOrd());
|
||||
for (long ord = Ordinals.MIN_ORDINAL; ord < leftOrds.getMaxOrd(); ++ord) {
|
||||
assertEquals(left.getMaxOrd(), right.getMaxOrd());
|
||||
for (long ord = BytesValues.WithOrdinals.MIN_ORDINAL; ord < left.getMaxOrd(); ++ord) {
|
||||
assertEquals(left.getValueByOrd(ord), right.getValueByOrd(ord));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -25,7 +25,6 @@ import org.apache.lucene.index.AtomicReaderContext;
|
|||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData.WithOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Random;
|
||||
|
@ -72,8 +71,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(2L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
|
||||
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
|
||||
}
|
||||
|
@ -84,8 +82,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(1L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("5"));
|
||||
}
|
||||
|
||||
|
@ -96,8 +93,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "med_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(2L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
|
||||
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
|
||||
}
|
||||
|
@ -109,8 +105,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "med_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(2L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
|
||||
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("100"));
|
||||
}
|
||||
|
@ -125,8 +120,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(1L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("100"));
|
||||
}
|
||||
}
|
||||
|
@ -169,8 +163,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(1L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(1L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("5"));
|
||||
}
|
||||
{
|
||||
|
@ -180,8 +173,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
|
|||
IndexFieldData<?> fieldData = getForField(fieldDataType, "high_freq");
|
||||
AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
|
||||
BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
|
||||
Docs ordinals = bytesValues.ordinals();
|
||||
assertThat(2L, equalTo(ordinals.getMaxOrd()));
|
||||
assertThat(2L, equalTo(bytesValues.getMaxOrd()));
|
||||
assertThat(bytesValues.getValueByOrd(0).utf8ToString(), equalTo("10"));
|
||||
assertThat(bytesValues.getValueByOrd(1).utf8ToString(), equalTo("5"));
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata.ordinals;
|
|||
|
||||
import org.apache.lucene.util.packed.PackedInts;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -103,7 +104,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
|
|||
}
|
||||
});
|
||||
Ordinals ords = creationMultiOrdinals(builder);
|
||||
Ordinals.Docs docs = ords.ordinals();
|
||||
BytesValues.WithOrdinals docs = ords.ordinals();
|
||||
int docId = ordsAndIds.get(0).id;
|
||||
List<Long> docOrds = new ArrayList<>();
|
||||
for (OrdAndId ordAndId : ordsAndIds) {
|
||||
|
@ -125,7 +126,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
|
|||
assertIter(docs, docId, array);
|
||||
}
|
||||
for (int i = docId + 1; i < ordAndId.id; i++) {
|
||||
assertThat(docs.getOrd(i), equalTo(Ordinals.MISSING_ORDINAL));
|
||||
assertThat(docs.getOrd(i), equalTo(BytesValues.WithOrdinals.MISSING_ORDINAL));
|
||||
}
|
||||
docId = ordAndId.id;
|
||||
docOrds.clear();
|
||||
|
@ -209,11 +210,11 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
|
|||
};
|
||||
|
||||
Ordinals ordinals = creationMultiOrdinals(builder);
|
||||
Ordinals.Docs docs = ordinals.ordinals();
|
||||
BytesValues.WithOrdinals docs = ordinals.ordinals();
|
||||
assertEquals(docs, ordinalPlan);
|
||||
}
|
||||
|
||||
protected static void assertIter(Ordinals.Docs docs, int docId, long... expectedOrdinals) {
|
||||
protected static void assertIter(BytesValues.WithOrdinals docs, int docId, long... expectedOrdinals) {
|
||||
assertThat(docs.setDocument(docId), equalTo(expectedOrdinals.length));
|
||||
for (long expectedOrdinal : expectedOrdinals) {
|
||||
assertThat(docs.nextOrd(), equalTo(expectedOrdinal));
|
||||
|
@ -261,11 +262,11 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
|
|||
};
|
||||
|
||||
Ordinals ordinals = new MultiOrdinals(builder, PackedInts.FASTEST);
|
||||
Ordinals.Docs docs = ordinals.ordinals();
|
||||
BytesValues.WithOrdinals docs = ordinals.ordinals();
|
||||
assertEquals(docs, ordinalPlan);
|
||||
}
|
||||
|
||||
private void assertEquals(Ordinals.Docs docs, long[][] ordinalPlan) {
|
||||
private void assertEquals(BytesValues.WithOrdinals docs, long[][] ordinalPlan) {
|
||||
long maxOrd = 0;
|
||||
for (int doc = 0; doc < ordinalPlan.length; ++doc) {
|
||||
if (ordinalPlan[doc].length > 0) {
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -26,7 +27,6 @@ import java.io.IOException;
|
|||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
|
@ -50,7 +50,7 @@ public class SingleOrdinalsTests extends ElasticsearchTestCase {
|
|||
|
||||
Ordinals ords = builder.build(ImmutableSettings.EMPTY);
|
||||
assertThat(ords, instanceOf(SinglePackedOrdinals.class));
|
||||
Ordinals.Docs docs = ords.ordinals();
|
||||
BytesValues.WithOrdinals docs = ords.ordinals();
|
||||
|
||||
for (Map.Entry<Integer, Long> entry : controlDocToOrdinal.entrySet()) {
|
||||
assertThat(entry.getValue(), equalTo(docs.getOrd(entry.getKey())));
|
||||
|
|
|
@ -26,9 +26,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
|
|||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.index.query.FilterBuilders;
|
||||
import org.elasticsearch.index.query.RangeFilterBuilder;
|
||||
import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
|
||||
|
@ -155,7 +153,6 @@ public class RandomTests extends ElasticsearchIntegrationTest {
|
|||
final IntOpenHashSet valuesSet = new IntOpenHashSet();
|
||||
cluster().wipeIndices("idx");
|
||||
prepareCreate("idx")
|
||||
.setSettings(ImmutableSettings.builder().put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, randomIntBetween(1, maxNumTerms)))
|
||||
.addMapping("type", jsonBuilder().startObject()
|
||||
.startObject("type")
|
||||
.startObject("properties")
|
||||
|
|
|
@ -37,6 +37,7 @@ public class FieldDataSourceTests extends ElasticsearchTestCase {
|
|||
final boolean multiValued = randomBoolean();
|
||||
final int maxLength = rarely() ? 3 : 10;
|
||||
return new BytesValues(multiValued) {
|
||||
private final BytesRef scratch = new BytesRef();
|
||||
BytesRef previous;
|
||||
@Override
|
||||
public int setDocument(int docId) {
|
||||
|
|
|
@ -63,7 +63,6 @@ import org.elasticsearch.index.cache.filter.FilterCacheModule;
|
|||
import org.elasticsearch.index.cache.filter.none.NoneFilterCache;
|
||||
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
|
||||
import org.elasticsearch.index.engine.IndexEngineModule;
|
||||
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.node.internal.InternalNode;
|
||||
import org.elasticsearch.plugins.PluginsService;
|
||||
|
@ -317,7 +316,6 @@ public final class InternalTestCluster extends TestCluster {
|
|||
}
|
||||
}
|
||||
builder.put("plugins.isolation", random.nextBoolean());
|
||||
builder.put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, 1 + random.nextInt(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_DEFAULT));
|
||||
if (random.nextInt(10) == 0) {
|
||||
builder.put(EsExecutors.PROCESSORS, 1 + random.nextInt(AbstractRandomizedTest.TESTS_PROCESSORS));
|
||||
} else {
|
||||
|
|
Loading…
Reference in New Issue