Added sparse multi ordinals implementation for field data.
This commit is contained in:
Martijn van Groningen
parent
9e79f54cb1
commit
346422b747
|
|
@ -21,11 +21,30 @@ package org.elasticsearch.index.fielddata.ordinals;
|
|||
|
||||
import org.elasticsearch.index.fielddata.util.IntArrayRef;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A thread safe ordinals abstraction. Ordinals can only be positive integers.
|
||||
*/
|
||||
public interface Ordinals {
|
||||
|
||||
static class Factories {
|
||||
|
||||
public static Ordinals createFromFlatOrdinals(int[][] ordinals, int numOrds, Map<String, String> options) {
|
||||
String multiOrdinals = options.get("multi_ordinals");
|
||||
if ("flat".equals(multiOrdinals)) {
|
||||
return new MultiFlatArrayOrdinals(ordinals, numOrds);
|
||||
}
|
||||
int multiOrdinalsMaxDocs = 16777216; // Equal to 64MB per storage array
|
||||
String multiOrdinalsMaxDocsVal = options.get("multi_ordinals_max_docs");
|
||||
if (multiOrdinalsMaxDocsVal != null) {
|
||||
multiOrdinalsMaxDocs = Integer.valueOf(multiOrdinalsMaxDocsVal);
|
||||
}
|
||||
|
||||
return new SparseMultiArrayOrdinals(ordinals, numOrds, multiOrdinalsMaxDocs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Are the ordinals backed by a single ordinals array?
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,327 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.ordinals;
|
||||
|
||||
import gnu.trove.list.array.TIntArrayList;
|
||||
import org.elasticsearch.ElasticSearchException;
|
||||
import org.elasticsearch.common.RamUsage;
|
||||
import org.elasticsearch.index.fielddata.util.IntArrayRef;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Ordinals implementation that stores the ordinals into sparse fixed arrays.
|
||||
* <p/>
|
||||
* This prevents large ordinal arrays that are created in for example {@link MultiFlatArrayOrdinals} when
|
||||
* only a few documents have a lot of terms per field.
|
||||
*/
|
||||
public class SparseMultiArrayOrdinals implements Ordinals {
|
||||
|
||||
// Contains pointer in starageOrdinals or the actual ordinal if document has one ordinal
|
||||
private final int[] lookup;
|
||||
|
||||
// Contains the ordinals for documents that have more than one ordinal. Each of this document has a start
|
||||
// point this array and the last ordinal is marked as negative.
|
||||
private final int[][] storageOrdinals;
|
||||
|
||||
// The n-th bit to shift the index of the storage array to inside the lookup pointer
|
||||
private final int storageShift;
|
||||
private final int numOrds;
|
||||
private final int numDocs;
|
||||
|
||||
private long size;
|
||||
|
||||
/**
|
||||
* @param loadedOrds The ordinals
|
||||
* @param numOrds The total number of unique ords
|
||||
* @param maxSize The maximum size in elements for each individual storage array
|
||||
*/
|
||||
public SparseMultiArrayOrdinals(int[][] loadedOrds, int numOrds, int maxSize) {
|
||||
int maxDoc = loadedOrds[0].length;
|
||||
if (loadedOrds.length * loadedOrds[0].length < maxSize) {
|
||||
maxSize = loadedOrds.length * loadedOrds[0].length + 1;
|
||||
}
|
||||
|
||||
int tempMaxSize = maxSize;
|
||||
int storageShift = 0;
|
||||
while (tempMaxSize > 0) {
|
||||
storageShift++;
|
||||
tempMaxSize = tempMaxSize >> 1;
|
||||
}
|
||||
this.storageShift = storageShift;
|
||||
|
||||
this.lookup = new int[maxDoc];
|
||||
this.numDocs = loadedOrds[0].length;
|
||||
this.numOrds = numOrds;
|
||||
List<TIntArrayList> allStorageArrays = new ArrayList<TIntArrayList>();
|
||||
|
||||
TIntArrayList currentStorageArray = new TIntArrayList(maxSize);
|
||||
currentStorageArray.add(Integer.MIN_VALUE);
|
||||
|
||||
TIntArrayList currentDocOrs = new TIntArrayList();
|
||||
for (int doc = 0; doc < maxDoc; doc++) {
|
||||
currentDocOrs.clear();
|
||||
for (int[] currentOrds : loadedOrds) {
|
||||
int currentOrd = currentOrds[doc];
|
||||
if (currentOrd == 0) {
|
||||
break;
|
||||
}
|
||||
currentDocOrs.add(currentOrd);
|
||||
}
|
||||
|
||||
int currentStorageArrayOffset = currentStorageArray.size();
|
||||
if (currentStorageArrayOffset + currentDocOrs.size() >= maxSize) {
|
||||
if (currentDocOrs.size() >= maxSize) {
|
||||
throw new ElasticSearchException("Doc[" + doc + "] has " + currentDocOrs.size() + " ordinals, but it surpasses the limit of " + maxSize);
|
||||
}
|
||||
|
||||
allStorageArrays.add(currentStorageArray);
|
||||
currentStorageArray = new TIntArrayList(maxSize);
|
||||
currentStorageArray.add(Integer.MIN_VALUE);
|
||||
currentStorageArrayOffset = 1;
|
||||
}
|
||||
|
||||
int size = currentDocOrs.size();
|
||||
if (size == 0) {
|
||||
lookup[doc] = 0;
|
||||
} else if (size == 1) {
|
||||
lookup[doc] = currentDocOrs.get(0);
|
||||
} else {
|
||||
// Mark the last ordinal for this doc.
|
||||
currentDocOrs.set(currentDocOrs.size() - 1, -currentDocOrs.get(currentDocOrs.size() - 1));
|
||||
currentStorageArray.addAll(currentDocOrs);
|
||||
lookup[doc] = allStorageArrays.size() << storageShift; // The left side of storageShift is for index in main array
|
||||
lookup[doc] |= (currentStorageArrayOffset & ((1 << storageShift) - 1)); // The right side of storageShift is for index in ordinal array
|
||||
lookup[doc] = -lookup[doc]; // Mark this value as 'pointer' into ordinals array
|
||||
}
|
||||
}
|
||||
|
||||
if (!currentStorageArray.isEmpty()) {
|
||||
allStorageArrays.add(currentStorageArray);
|
||||
}
|
||||
|
||||
this.storageOrdinals = new int[allStorageArrays.size()][];
|
||||
for (int i = 0; i < this.storageOrdinals.length; i++) {
|
||||
this.storageOrdinals[i] = allStorageArrays.get(i).toArray();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasSingleArrayBackingStorage() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getBackingStorage() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
long size = 0;
|
||||
size += RamUsage.NUM_BYTES_ARRAY_HEADER;
|
||||
for (int[] ordinal : storageOrdinals) {
|
||||
size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
|
||||
}
|
||||
size += RamUsage.NUM_BYTES_ARRAY_HEADER + (RamUsage.NUM_BYTES_INT * lookup.length);
|
||||
this.size = size;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumDocs() {
|
||||
return numDocs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumOrds() {
|
||||
return numOrds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Docs ordinals() {
|
||||
return new Docs(this, lookup, storageOrdinals);
|
||||
}
|
||||
|
||||
static class Docs implements Ordinals.Docs {
|
||||
|
||||
private final SparseMultiArrayOrdinals parent;
|
||||
private final int[] lookup;
|
||||
private final int[][] ordinals;
|
||||
private final IterImpl iter;
|
||||
|
||||
private final IntArrayRef intsScratch;
|
||||
|
||||
public Docs(SparseMultiArrayOrdinals parent, int[] lookup, int[][] ordinals) {
|
||||
this.parent = parent;
|
||||
this.lookup = lookup;
|
||||
this.ordinals = ordinals;
|
||||
this.iter = new IterImpl(lookup, ordinals);
|
||||
this.intsScratch = new IntArrayRef(new int[parent.numOrds]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Ordinals ordinals() {
|
||||
return this.parent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumDocs() {
|
||||
return parent.getNumDocs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumOrds() {
|
||||
return parent.getNumOrds();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrd(int docId) {
|
||||
int pointer = lookup[docId];
|
||||
if (pointer == 0) {
|
||||
return 0;
|
||||
} else if (pointer > 0) {
|
||||
return pointer;
|
||||
} else {
|
||||
pointer = -pointer;
|
||||
int allOrdsIndex = pointer >> parent.storageShift;
|
||||
int ordsIndex = (pointer & ((1 << parent.storageShift) - 1));
|
||||
return ordinals[allOrdsIndex][ordsIndex];
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public IntArrayRef getOrds(int docId) {
|
||||
intsScratch.end = 0;
|
||||
int pointer = lookup[docId];
|
||||
// System.out.println("\nPointer: " + pointer);
|
||||
if (pointer == 0) {
|
||||
return IntArrayRef.EMPTY;
|
||||
} else if (pointer > 0) {
|
||||
intsScratch.end = 1;
|
||||
intsScratch.values[0] = pointer;
|
||||
return intsScratch;
|
||||
} else {
|
||||
pointer = -pointer;
|
||||
int allOrdsIndex = pointer >> parent.storageShift;
|
||||
int ordsIndex = (pointer & ((1 << parent.storageShift) - 1));
|
||||
// System.out.println("Storage index: " + allOrdsIndex);
|
||||
// System.out.println("Ordinal index: " + ordsIndex);
|
||||
|
||||
int[] ords = ordinals[allOrdsIndex];
|
||||
|
||||
int i = 0;
|
||||
int ord;
|
||||
while ((ord = ords[ordsIndex++]) > 0) {
|
||||
intsScratch.values[i++] = ord;
|
||||
}
|
||||
intsScratch.values[i++] = -ord;
|
||||
intsScratch.end = i;
|
||||
|
||||
return intsScratch;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iter getIter(int docId) {
|
||||
return iter.reset(docId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
|
||||
int pointer = lookup[docId];
|
||||
if (pointer >= 0) {
|
||||
proc.onOrdinal(docId, pointer);
|
||||
} else {
|
||||
pointer = -pointer;
|
||||
int allOrdsIndex = pointer >> parent.storageShift;
|
||||
int ordsIndex = (pointer & ((1 << parent.storageShift) - 1));
|
||||
int[] ords = ordinals[allOrdsIndex];
|
||||
int i = ordsIndex;
|
||||
for (; ords[i] > 0; i++) {
|
||||
proc.onOrdinal(docId, ords[i]);
|
||||
}
|
||||
proc.onOrdinal(docId, -ords[i]);
|
||||
}
|
||||
}
|
||||
|
||||
class IterImpl implements Docs.Iter {
|
||||
|
||||
private final int[] lookup;
|
||||
private final int[][] ordinals;
|
||||
|
||||
private int pointer;
|
||||
|
||||
private int allOrdsIndex;
|
||||
private int ordsIndex;
|
||||
|
||||
private int ord;
|
||||
|
||||
public IterImpl(int[] lookup, int[][] ordinals) {
|
||||
this.lookup = lookup;
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
public IterImpl reset(int docId) {
|
||||
pointer = lookup[docId];
|
||||
if (pointer < 0) {
|
||||
int pointer = -this.pointer;
|
||||
allOrdsIndex = pointer >> parent.storageShift;
|
||||
ordsIndex = (pointer & ((1 << parent.storageShift) - 1));
|
||||
ord = ordinals[allOrdsIndex][ordsIndex];
|
||||
} else {
|
||||
ord = pointer;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int next() {
|
||||
if (ord <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (pointer > 0) {
|
||||
ord = 0;
|
||||
return pointer;
|
||||
} else {
|
||||
ord = ordinals[allOrdsIndex][ordsIndex++];
|
||||
if (ord < 0) {
|
||||
return -ord;
|
||||
}
|
||||
return ord;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.ByteValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new ByteArrayAtomicFieldData.WithOrdinals(values.toArray(new byte[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new ByteArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new byte[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
|
@ -143,7 +143,10 @@ public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<Concr
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new ConcreteBytesRefAtomicFieldData(
|
||||
values.toArray(new BytesRef[values.size()]),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new DoubleArrayAtomicFieldData.WithOrdinals(values.toArray(new double[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new DoubleArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new double[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new FloatArrayAtomicFieldData.WithOrdinals(values.toArray(new float[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new FloatArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new float[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -149,7 +149,12 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon.toArray(new double[lon.size()]), lat.toArray(new double[lat.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(
|
||||
lon.toArray(new double[lon.size()]),
|
||||
lat.toArray(new double[lat.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.IntValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new IntArrayAtomicFieldData.WithOrdinals(values.toArray(new int[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new IntArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new int[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new LongArrayAtomicFieldData.WithOrdinals(values.toArray(new long[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new LongArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new long[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
|
@ -175,7 +175,11 @@ public class PagedBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesA
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new PagedBytesAtomicFieldData(
|
||||
bytesReader,
|
||||
termOrdToBytesOffsetReader,
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.ShortValuesComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
|
|
@ -146,7 +146,11 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
|
|||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new ShortArrayAtomicFieldData.WithOrdinals(values.toArray(new short[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new ShortArrayAtomicFieldData.WithOrdinals(
|
||||
values.toArray(new short[values.size()]),
|
||||
reader.maxDoc(),
|
||||
Ordinals.Factories.createFromFlatOrdinals(nativeOrdinals, termOrd, fieldDataType.getOptions())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata.ordinals;
|
||||
|
||||
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class FlatMultiOrdinalsTests extends MultiOrdinalsTests {
|
||||
|
||||
@Override
|
||||
protected Ordinals creationMultiOrdinals(int[][] ordinals, int maxOrds) {
|
||||
return new MultiFlatArrayOrdinals(ordinals, maxOrds);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata.ordinals;
|
||||
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.util.IntArrayRef;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
*/
|
||||
public abstract class MultiOrdinalsTests {
|
||||
|
||||
protected abstract Ordinals creationMultiOrdinals(int[][] ordinals, int maxOrds);
|
||||
|
||||
@Test
|
||||
public void testOrdinals() throws Exception {
|
||||
int maxDoc = 7;
|
||||
int maxOrds = 32;
|
||||
int[][] ords = new int[maxOrds][maxDoc];
|
||||
// Doc 1
|
||||
ords[0][0] = 2;
|
||||
ords[1][0] = 4;
|
||||
|
||||
// Doc 2
|
||||
ords[0][1] = 1;
|
||||
|
||||
// Doc 3
|
||||
ords[0][2] = 3;
|
||||
|
||||
// Doc 4
|
||||
|
||||
// Doc 5
|
||||
ords[0][4] = 1;
|
||||
ords[1][4] = 3;
|
||||
ords[2][4] = 4;
|
||||
ords[3][4] = 5;
|
||||
ords[4][4] = 6;
|
||||
|
||||
// Doc 6
|
||||
for (int i = 0; i < maxOrds; i++) {
|
||||
ords[i][5] = (i + 1);
|
||||
}
|
||||
|
||||
// Doc 7
|
||||
for (int i = 0; i < maxOrds; i++) {
|
||||
ords[i][6] = (i + 1);
|
||||
}
|
||||
|
||||
Ordinals ordinals = creationMultiOrdinals(ords, maxOrds);
|
||||
Ordinals.Docs docs = ordinals.ordinals();
|
||||
assertThat(docs.getNumDocs(), equalTo(maxDoc));
|
||||
assertThat(docs.getNumOrds(), equalTo(maxOrds)); // Includes null ord
|
||||
assertThat(docs.isMultiValued(), equalTo(true));
|
||||
|
||||
// Document 1
|
||||
assertThat(docs.getOrd(0), equalTo(2));
|
||||
IntArrayRef ref = docs.getOrds(0);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(2));
|
||||
assertThat(ref.values[1], equalTo(4));
|
||||
assertThat(ref.end, equalTo(2));
|
||||
assertIter(docs.getIter(0), 2, 4);
|
||||
docs.forEachOrdinalInDoc(0, assertOrdinalInProcDoc(2, 4));
|
||||
|
||||
// Document 2
|
||||
assertThat(docs.getOrd(1), equalTo(1));
|
||||
ref = docs.getOrds(1);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(1));
|
||||
assertThat(ref.end, equalTo(1));
|
||||
assertIter(docs.getIter(1), 1);
|
||||
docs.forEachOrdinalInDoc(1, assertOrdinalInProcDoc(1));
|
||||
|
||||
// Document 3
|
||||
assertThat(docs.getOrd(2), equalTo(3));
|
||||
ref = docs.getOrds(2);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(3));
|
||||
assertThat(ref.end, equalTo(1));
|
||||
assertIter(docs.getIter(2), 3);
|
||||
docs.forEachOrdinalInDoc(2, assertOrdinalInProcDoc(3));
|
||||
|
||||
// Document 4
|
||||
assertThat(docs.getOrd(3), equalTo(0));
|
||||
ref = docs.getOrds(3);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.end, equalTo(0));
|
||||
assertIter(docs.getIter(3));
|
||||
docs.forEachOrdinalInDoc(3, assertOrdinalInProcDoc(0));
|
||||
|
||||
// Document 5
|
||||
assertThat(docs.getOrd(4), equalTo(1));
|
||||
ref = docs.getOrds(4);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(1));
|
||||
assertThat(ref.values[1], equalTo(3));
|
||||
assertThat(ref.values[2], equalTo(4));
|
||||
assertThat(ref.values[3], equalTo(5));
|
||||
assertThat(ref.values[4], equalTo(6));
|
||||
assertThat(ref.end, equalTo(5));
|
||||
assertIter(docs.getIter(4), 1, 3, 4, 5, 6);
|
||||
docs.forEachOrdinalInDoc(4, assertOrdinalInProcDoc(1, 3, 4, 5, 6));
|
||||
|
||||
// Document 6
|
||||
assertThat(docs.getOrd(5), equalTo(1));
|
||||
ref = docs.getOrds(5);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
int[] expectedOrds = new int[maxOrds];
|
||||
for (int i = 0; i < maxOrds; i++) {
|
||||
expectedOrds[i] = i + 1;
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertIter(docs.getIter(5), expectedOrds);
|
||||
docs.forEachOrdinalInDoc(5, assertOrdinalInProcDoc(expectedOrds));
|
||||
assertThat(ref.end, equalTo(maxOrds));
|
||||
|
||||
// Document 7
|
||||
assertThat(docs.getOrd(6), equalTo(1));
|
||||
ref = docs.getOrds(6);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
expectedOrds = new int[maxOrds];
|
||||
for (int i = 0; i < maxOrds; i++) {
|
||||
expectedOrds[i] = i + 1;
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertIter(docs.getIter(6), expectedOrds);
|
||||
docs.forEachOrdinalInDoc(6, assertOrdinalInProcDoc(expectedOrds));
|
||||
assertThat(ref.end, equalTo(maxOrds));
|
||||
}
|
||||
|
||||
protected static void assertIter(Ordinals.Docs.Iter iter, int... expectedOrdinals) {
|
||||
for (int expectedOrdinal : expectedOrdinals) {
|
||||
assertThat(iter.next(), equalTo(expectedOrdinal));
|
||||
}
|
||||
assertThat(iter.next(), equalTo(0)); // Last one should always be 0
|
||||
assertThat(iter.next(), equalTo(0)); // Just checking it stays 0
|
||||
}
|
||||
|
||||
protected static Ordinals.Docs.OrdinalInDocProc assertOrdinalInProcDoc(int... expectedOrdinals) {
|
||||
return new AssertingOrdinalInDocProc(expectedOrdinals);
|
||||
}
|
||||
|
||||
static class AssertingOrdinalInDocProc implements Ordinals.Docs.OrdinalInDocProc {
|
||||
|
||||
private final int[] expectedOrdinals;
|
||||
private int index = 0;
|
||||
|
||||
AssertingOrdinalInDocProc(int... expectedOrdinals) {
|
||||
this.expectedOrdinals = expectedOrdinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOrdinal(int docId, int ordinal) {
|
||||
assertThat(ordinal, equalTo(expectedOrdinals[index++]));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata.ordinals;
|
||||
|
||||
import org.elasticsearch.ElasticSearchException;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.SparseMultiArrayOrdinals;
|
||||
import org.elasticsearch.index.fielddata.util.IntArrayRef;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.testng.Assert.fail;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class SparseMultiOrdinalsTests extends MultiOrdinalsTests {
|
||||
|
||||
@Override
|
||||
protected Ordinals creationMultiOrdinals(int[][] ordinals, int maxOrds) {
|
||||
return new SparseMultiArrayOrdinals(ordinals, maxOrds, 64);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValuesSurpassOrdinalsLimit() throws Exception {
|
||||
int maxDoc = 2;
|
||||
int maxOrds = 128;
|
||||
int[][] ords = new int[maxOrds][maxDoc];
|
||||
// Doc 1
|
||||
ords[0][0] = 2;
|
||||
ords[1][0] = 4;
|
||||
|
||||
// Doc 2
|
||||
for (int i = 0; i < maxOrds; i++) {
|
||||
ords[i][1] = (i + 1);
|
||||
}
|
||||
|
||||
try {
|
||||
creationMultiOrdinals(ords, maxOrds);
|
||||
fail("Exception should have been throwed");
|
||||
} catch (ElasticSearchException e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValuesDocsWithOverlappingStorageArrays() throws Exception {
|
||||
int maxDoc = 7;
|
||||
int maxOrds = 15;
|
||||
int[][] ords = new int[maxOrds][maxDoc];
|
||||
// Doc 1
|
||||
for (int i = 0; i < 10; i++) {
|
||||
ords[i][0] = (i + 1);
|
||||
}
|
||||
|
||||
// Doc 2
|
||||
for (int i = 0; i < 15; i++) {
|
||||
ords[i][1] = (i + 1);
|
||||
}
|
||||
|
||||
// Doc 3
|
||||
ords[0][2] = 1;
|
||||
|
||||
// Doc 4
|
||||
for (int i = 0; i < 5; i++) {
|
||||
ords[i][3] = (i + 1);
|
||||
}
|
||||
|
||||
// Doc 5
|
||||
for (int i = 0; i < 6; i++) {
|
||||
ords[i][4] = (i + 1);
|
||||
}
|
||||
|
||||
// Doc 6
|
||||
ords[0][5] = 2;
|
||||
|
||||
// Doc 7
|
||||
for (int i = 0; i < 10; i++) {
|
||||
ords[i][6] = (i + 1);
|
||||
}
|
||||
|
||||
Ordinals ordinals = new SparseMultiArrayOrdinals(ords, maxOrds, 20);
|
||||
Ordinals.Docs docs = ordinals.ordinals();
|
||||
assertThat(docs.getNumDocs(), equalTo(maxDoc));
|
||||
assertThat(docs.getNumOrds(), equalTo(maxOrds)); // Includes null ord
|
||||
assertThat(docs.isMultiValued(), equalTo(true));
|
||||
|
||||
// Document 1
|
||||
assertThat(docs.getOrd(0), equalTo(1));
|
||||
IntArrayRef ref = docs.getOrds(0);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertThat(ref.end, equalTo(10));
|
||||
|
||||
// Document 2
|
||||
assertThat(docs.getOrd(1), equalTo(1));
|
||||
ref = docs.getOrds(1);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
for (int i = 0; i < 15; i++) {
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertThat(ref.end, equalTo(15));
|
||||
|
||||
// Document 3
|
||||
assertThat(docs.getOrd(2), equalTo(1));
|
||||
ref = docs.getOrds(2);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(1));
|
||||
assertThat(ref.end, equalTo(1));
|
||||
|
||||
// Document 4
|
||||
assertThat(docs.getOrd(3), equalTo(1));
|
||||
ref = docs.getOrds(3);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertThat(ref.end, equalTo(5));
|
||||
|
||||
// Document 5
|
||||
assertThat(docs.getOrd(4), equalTo(1));
|
||||
ref = docs.getOrds(4);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
for (int i = 0; i < 6; i++) {
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertThat(ref.end, equalTo(6));
|
||||
|
||||
// Document 6
|
||||
assertThat(docs.getOrd(5), equalTo(2));
|
||||
ref = docs.getOrds(5);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
assertThat(ref.values[0], equalTo(2));
|
||||
assertThat(ref.end, equalTo(1));
|
||||
|
||||
// Document 7
|
||||
assertThat(docs.getOrd(6), equalTo(1));
|
||||
ref = docs.getOrds(6);
|
||||
assertThat(ref.start, equalTo(0));
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertThat(ref.values[i], equalTo(i + 1));
|
||||
}
|
||||
assertThat(ref.end, equalTo(10));
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue