move sorting to work with new field data
This commit is contained in:
Shay Banon
parent
b739bf97d4
commit
5b7173fc35
|
|
@ -32,7 +32,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
|
|||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.index.analysis.AnalyzerScope;
|
||||
import org.elasticsearch.index.analysis.NamedAnalyzer;
|
||||
import org.elasticsearch.index.field.data.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
|
|
@ -215,7 +215,7 @@ public class Lucene {
|
|||
out.writeString(sortField.getField());
|
||||
}
|
||||
if (sortField.getComparatorSource() != null) {
|
||||
writeSortType(out, ((FieldDataType.ExtendedFieldComparatorSource) sortField.getComparatorSource()).reducedType());
|
||||
writeSortType(out, ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()).reducedType());
|
||||
} else {
|
||||
writeSortType(out, sortField.getType());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
|
||||
buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
|
||||
.put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "packed_bytes"), new PackedBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "paged_bytes"), new PagesBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new ByteArrayAtomicFieldData.Single(new byte[0], 0);
|
||||
return new ByteArrayAtomicFieldData.SingleFixedSet(new byte[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
|
|||
} else {
|
||||
ret.bytes = value.bytes;
|
||||
ret.offset = value.offset;
|
||||
ret.length = value.offset;
|
||||
ret.length = value.length;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
|
||||
return new DoubleArrayAtomicFieldData.SingleFixedSet(new double[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new FloatArrayAtomicFieldData.Single(new float[0], 0);
|
||||
return new FloatArrayAtomicFieldData.SingleFixedSet(new float[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new GeoPointDoubleArrayAtomicFieldData.Single(new double[0], new double[0], 0);
|
||||
return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(new double[1], new double[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new IntArrayAtomicFieldData.Single(new int[0], 0);
|
||||
return new IntArrayAtomicFieldData.SingleFixedSet(new int[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new LongArrayAtomicFieldData.Single(new long[0], 0);
|
||||
return new LongArrayAtomicFieldData.SingleFixedSet(new long[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
|
|||
|
||||
/**
|
||||
*/
|
||||
public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
|
||||
public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
|
||||
|
||||
// 0 ordinal in values means no value (its null)
|
||||
private final PagedBytes.Reader bytes;
|
||||
|
|
@ -42,7 +42,7 @@ public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<Script
|
|||
private int[] hashes;
|
||||
private long size = -1;
|
||||
|
||||
public PackedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
|
||||
public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
|
||||
this.bytes = bytes;
|
||||
this.termOrdToBytesOffset = termOrdToBytesOffset;
|
||||
this.ordinals = ordinals;
|
||||
|
|
@ -41,17 +41,17 @@ import java.util.ArrayList;
|
|||
|
||||
/**
|
||||
*/
|
||||
public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedBytesAtomicFieldData> implements IndexOrdinalFieldData<PackedBytesAtomicFieldData> {
|
||||
public class PagesBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesAtomicFieldData> implements IndexOrdinalFieldData<PagedBytesAtomicFieldData> {
|
||||
|
||||
public static class Builder implements IndexFieldData.Builder {
|
||||
|
||||
@Override
|
||||
public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
|
||||
return new PackedBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
|
||||
return new PagesBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
|
||||
}
|
||||
}
|
||||
|
||||
public PackedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
|
||||
public PagesBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
|
||||
super(index, indexSettings, fieldNames, fieldDataType, cache);
|
||||
}
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
|
|||
}
|
||||
|
||||
@Override
|
||||
public PackedBytesAtomicFieldData load(AtomicReaderContext context) {
|
||||
public PagedBytesAtomicFieldData load(AtomicReaderContext context) {
|
||||
try {
|
||||
return cache.load(context, this);
|
||||
} catch (Throwable e) {
|
||||
|
|
@ -74,7 +74,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
|
|||
}
|
||||
|
||||
@Override
|
||||
public PackedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
|
||||
public PagedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
|
||||
AtomicReader reader = context.reader();
|
||||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
|
|
@ -83,7 +83,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
|
|||
// 0 is reserved for "unset"
|
||||
bytes.copyUsingLengthPrefix(new BytesRef());
|
||||
GrowableWriter termOrdToBytesOffset = new GrowableWriter(1, 2, PackedInts.FASTEST);
|
||||
return new PackedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
|
||||
return new PagedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
|
||||
}
|
||||
|
||||
final PagedBytes bytes = new PagedBytes(15);
|
||||
|
|
@ -174,13 +174,13 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
|
|||
PackedInts.Reader termOrdToBytesOffsetReader = termOrdToBytesOffset.getMutable();
|
||||
|
||||
if (ordinals.size() == 1) {
|
||||
return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
|
||||
return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
|
||||
} else {
|
||||
int[][] nativeOrdinals = new int[ordinals.size()][];
|
||||
for (int i = 0; i < nativeOrdinals.length; i++) {
|
||||
nativeOrdinals[i] = ordinals.get(i);
|
||||
}
|
||||
return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -83,7 +83,7 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
|
|||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return new ShortArrayAtomicFieldData.Single(new short[0], 0);
|
||||
return new ShortArrayAtomicFieldData.SingleFixedSet(new short[1], 0, new FixedBitSet(1));
|
||||
}
|
||||
|
||||
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ public class SortParseElement implements SearchParseElement {
|
|||
}
|
||||
throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on");
|
||||
}
|
||||
sortFields.add(new SortField(fieldMapper.names().indexName(), fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache(), missing), reverse));
|
||||
sortFields.add(new SortField(fieldMapper.names().indexName(), context.fieldData().getForField(fieldMapper).comparatorSource(missing), reverse));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -531,22 +531,24 @@ public class SimpleSortTests extends AbstractNodesTests {
|
|||
|
||||
client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet();
|
||||
|
||||
logger.info("--> sort with no missing");
|
||||
logger.info("--> sort with no missing (same as missing _last)");
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
|
||||
assertThat(searchResponse.hits().getAt(1).id(), equalTo("2"));
|
||||
assertThat(searchResponse.hits().getAt(2).id(), equalTo("3"));
|
||||
assertThat(searchResponse.hits().getAt(1).id(), equalTo("3"));
|
||||
assertThat(searchResponse.hits().getAt(2).id(), equalTo("2"));
|
||||
|
||||
logger.info("--> sort with missing _last");
|
||||
searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last"))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
|
||||
|
|
@ -558,6 +560,7 @@ public class SimpleSortTests extends AbstractNodesTests {
|
|||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first"))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), equalTo("2"));
|
||||
|
|
|
|||
|
|
@ -26,10 +26,10 @@ import org.testng.annotations.Test;
|
|||
/**
|
||||
*/
|
||||
@Test
|
||||
public class PackedBytesStringFieldDataTests extends StringFieldDataTests {
|
||||
public class PagedBytesStringFieldDataTests extends StringFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
return new FieldDataType("string", "packed_bytes", ImmutableMap.<String, String>of());
|
||||
return new FieldDataType("string", "paged_bytes", ImmutableMap.<String, String>of());
|
||||
}
|
||||
}
|
||||
|
|
@ -56,6 +56,13 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
|
|||
return "4";
|
||||
}
|
||||
|
||||
private String toString(Object value) {
|
||||
if (value instanceof BytesRef) {
|
||||
return ((BytesRef) value).utf8ToString();
|
||||
}
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
protected void fillSingleValueAllSet() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
|
|
@ -218,8 +225,11 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
|
|||
new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));
|
||||
|
|
|
|||
Loading…
Reference in New Issue