honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow binary sort values. #17959 

The `ip` field uses a binary representation internally. This breaks when
rendering sort values in search responses since elasticsearch tries to write a
binary byte[] as an utf8 json string. This commit extends the `DocValueFormat`
API in order to give fields a chance to choose how to render values.

Closes #6077
This commit is contained in:
Adrien Grand 2016-04-25 17:45:33 +02:00
parent c55df195c5
commit de8354dd7f
39 changed files with 402 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.index.query;
import org.apache.lucene.search.Sort;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
@ -41,6 +40,7 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.highlight.HighlightBuilder;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.sort.SortBuilder;
import java.io.IOException;
@ -512,7 +512,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
innerHitsContext.fetchSourceContext(fetchSourceContext);
}
if (sorts != null) {
Optional<Sort> optionalSort = SortBuilder.buildSort(sorts, context);
Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, context);
if (optionalSort.isPresent()) {
innerHitsContext.sort(optionalSort.get());
}

58
core/src/main/java/org/elasticsearch/search/DocValueFormat.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.search;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.io.stream.NamedWriteable;
@ -29,8 +28,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.index.mapper.ip.IpFieldMapper;
import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper;
import org.joda.time.DateTimeZone;
@ -48,16 +47,33 @@ import java.util.concurrent.Callable;
/** A formatter for values as returned by the fielddata/doc-values APIs. */
public interface DocValueFormat extends NamedWriteable {
/** Format a long value. This is used by terms and histogram aggregations
* to format keys for fields that use longs as a doc value representation
* such as the {@code long} and {@code date} fields. */
String format(long value);
/** Format a double value. This is used by terms and stats aggregations
* to format keys for fields that use numbers as a doc value representation
* such as the {@code long}, {@code double} or {@code date} fields. */
String format(double value);
/** Format a double value. This is used by terms aggregations to format
* keys for fields that use binary doc value representations such as the
* {@code keyword} and {@code ip} fields. */
String format(BytesRef value);
/** Parse a value that was formatted with {@link #format(long)} back to the
* original long value. */
long parseLong(String value, boolean roundUp, Callable<Long> now);
/** Parse a value that was formatted with {@link #format(double)} back to
* the original double value. */
double parseDouble(String value, boolean roundUp, Callable<Long> now);
/** Parse a value that was formatted with {@link #format(BytesRef)} back
* to the original BytesRef. */
BytesRef parseBytesRef(String value);
public static final DocValueFormat RAW = new DocValueFormat() {
@Override
@ -81,7 +97,7 @@ public interface DocValueFormat extends NamedWriteable {
@Override
public String format(BytesRef value) {
return Term.toString(value);
return value.utf8ToString();
}
@Override
@ -99,6 +115,10 @@ public interface DocValueFormat extends NamedWriteable {
public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
return Double.parseDouble(value);
}
public BytesRef parseBytesRef(String value) {
return new BytesRef(value);
}
};
public static final class DateTime implements DocValueFormat {
@ -154,6 +174,11 @@ public interface DocValueFormat extends NamedWriteable {
public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
return parseLong(value, roundUp, now);
}
@Override
public BytesRef parseBytesRef(String value) {
throw new UnsupportedOperationException();
}
}
public static final DocValueFormat GEOHASH = new DocValueFormat() {
@ -191,6 +216,11 @@ public interface DocValueFormat extends NamedWriteable {
public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef parseBytesRef(String value) {
throw new UnsupportedOperationException();
}
};
public static final DocValueFormat BOOLEAN = new DocValueFormat() {
@ -221,13 +251,24 @@ public interface DocValueFormat extends NamedWriteable {
@Override
public long parseLong(String value, boolean roundUp, Callable<Long> now) {
throw new UnsupportedOperationException();
switch (value) {
case "false":
return 0;
case "true":
return 1;
}
throw new IllegalArgumentException("Cannot parse boolean [" + value + "], expected either [true] or [false]");
}
@Override
public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef parseBytesRef(String value) {
throw new UnsupportedOperationException();
}
};
public static final DocValueFormat IP = new DocValueFormat() {
@ -268,6 +309,11 @@ public interface DocValueFormat extends NamedWriteable {
public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
return parseLong(value, roundUp, now);
}
@Override
public BytesRef parseBytesRef(String value) {
return new BytesRef(InetAddressPoint.encode(InetAddresses.forString(value)));
}
};
public static final class Decimal implements DocValueFormat {
@ -344,5 +390,9 @@ public interface DocValueFormat extends NamedWriteable {
return n.doubleValue();
}
@Override
public BytesRef parseBytesRef(String value) {
throw new UnsupportedOperationException();
}
}
}

3
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -104,6 +104,7 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
import org.elasticsearch.search.query.ScrollQuerySearchResult;
import org.elasticsearch.search.rescore.RescoreBuilder;
import org.elasticsearch.search.searchafter.SearchAfterBuilder;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.suggest.Suggesters;
import org.elasticsearch.threadpool.ThreadPool;
@ -698,7 +699,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
}
if (source.sorts() != null) {
try {
Optional<Sort> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
if (optionalSort.isPresent()) {
context.sort(optionalSort.get());
}

16
core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
@ -45,6 +44,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.InternalSearchHits;
import org.elasticsearch.search.internal.SubSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import java.io.IOException;
import java.util.List;
@ -78,9 +78,9 @@ public class TopHitsAggregator extends MetricsAggregator {
@Override
public boolean needsScores() {
Sort sort = subSearchContext.sort();
SortAndFormats sort = subSearchContext.sort();
if (sort != null) {
return sort.needsScores() || subSearchContext.trackScores();
return sort.sort.needsScores() || subSearchContext.trackScores();
} else {
// sort by score
return true;
@ -112,12 +112,12 @@ public class TopHitsAggregator extends MetricsAggregator {
public void collect(int docId, long bucket) throws IOException {
TopDocsAndLeafCollector collectors = topDocsCollectors.get(bucket);
if (collectors == null) {
Sort sort = subSearchContext.sort();
SortAndFormats sort = subSearchContext.sort();
int topN = subSearchContext.from() + subSearchContext.size();
// In the QueryPhase we don't need this protection, because it is build into the IndexSearcher,
// but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
collectors = new TopDocsAndLeafCollector(topLevelCollector);
collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx);
collectors.leafCollector.setScorer(scorer);
@ -137,7 +137,7 @@ public class TopHitsAggregator extends MetricsAggregator {
} else {
final TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs();
subSearchContext.queryResult().topDocs(topDocs);
subSearchContext.queryResult().topDocs(topDocs, subSearchContext.sort() == null ? null : subSearchContext.sort().formats);
int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
@ -153,7 +153,7 @@ public class TopHitsAggregator extends MetricsAggregator {
searchHitFields.score(scoreDoc.score);
if (scoreDoc instanceof FieldDoc) {
FieldDoc fieldDoc = (FieldDoc) scoreDoc;
searchHitFields.sortValues(fieldDoc.fields);
searchHitFields.sortValues(fieldDoc.fields, subSearchContext.sort().formats);
}
}
topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(),
@ -166,7 +166,7 @@ public class TopHitsAggregator extends MetricsAggregator {
public InternalTopHits buildEmptyAggregation() {
TopDocs topDocs;
if (subSearchContext.sort() != null) {
topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().getSort(), Float.NaN);
topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().sort.getSort(), Float.NaN);
} else {
topDocs = Lucene.EMPTY_TOP_DOCS;
}

4
core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.search.aggregations.metrics.tophits;
import org.apache.lucene.search.Sort;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.aggregations.Aggregator;
@ -35,6 +34,7 @@ import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.highlight.HighlightBuilder;
import org.elasticsearch.search.internal.SubSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.sort.SortBuilder;
import java.io.IOException;
@ -87,7 +87,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
subSearchContext.from(from);
subSearchContext.size(size);
if (sorts != null) {
Optional<Sort> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
if (optionalSort.isPresent()) {
subSearchContext.sort(optionalSort.get());
}

2
core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
View File

@ -362,7 +362,7 @@ public class SearchPhaseController extends AbstractComponent {
if (sorted) {
FieldDoc fieldDoc = (FieldDoc) shardDoc;
searchHit.sortValues(fieldDoc.fields);
searchHit.sortValues(fieldDoc.fields, firstResult.sortValueFormats());
if (sortScoreIndex != -1) {
searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
}

4
core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
View File

@ -142,7 +142,7 @@ public final class InnerHitsContext {
TopDocsCollector topDocsCollector;
if (sort() != null) {
try {
topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores());
topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
} catch (IOException e) {
throw ExceptionsHelper.convertToElastic(e);
}
@ -317,7 +317,7 @@ public final class InnerHitsContext {
int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
TopDocsCollector topDocsCollector;
if (sort() != null) {
topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores());
topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
} else {
topDocsCollector = TopScoreDocCollector.create(topN);
}

4
core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
View File

@ -73,7 +73,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
} catch (IOException e) {
throw ExceptionsHelper.convertToElastic(e);
}
innerHits.queryResult().topDocs(topDocs);
innerHits.queryResult().topDocs(topDocs, innerHits.sort() == null ? null : innerHits.sort().formats);
int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
@ -89,7 +89,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
searchHitFields.score(scoreDoc.score);
if (scoreDoc instanceof FieldDoc) {
FieldDoc fieldDoc = (FieldDoc) scoreDoc;
searchHitFields.sortValues(fieldDoc.fields);
searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats);
}
}
results.put(entry.getKey(), fetchResult.hits());

8
core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
View File

@ -25,7 +25,6 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
@ -71,6 +70,7 @@ import org.elasticsearch.search.profile.Profilers;
import org.elasticsearch.search.query.QueryPhaseExecutionException;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import java.io.IOException;
@ -114,7 +114,7 @@ public class DefaultSearchContext extends SearchContext {
private FetchSourceContext fetchSourceContext;
private int from = -1;
private int size = -1;
private Sort sort;
private SortAndFormats sort;
private Float minimumScore;
private boolean trackScores = false; // when sorting, track scores as well...
private FieldDoc searchAfter;
@ -532,13 +532,13 @@ public class DefaultSearchContext extends SearchContext {
}
@Override
public SearchContext sort(Sort sort) {
public SearchContext sort(SortAndFormats sort) {
this.sort = sort;
return this;
}
@Override
public Sort sort() {
public SortAndFormats sort() {
return this.sort;
}

6
core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.search.internal;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Counter;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cache.recycler.PageCacheRecycler;
@ -55,6 +54,7 @@ import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.profile.Profilers;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import java.util.List;
@ -306,12 +306,12 @@ public abstract class FilteredSearchContext extends SearchContext {
}
@Override
public SearchContext sort(Sort sort) {
public SearchContext sort(SortAndFormats sort) {
return in.sort(sort);
}
@Override
public Sort sort() {
public SortAndFormats sort() {
return in.sort();
}

26
core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.io.stream.StreamInput;
@ -34,6 +33,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
@ -44,6 +44,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@ -326,21 +327,13 @@ public class InternalSearchHit implements SearchHit {
this.highlightFields = highlightFields;
}
public void sortValues(Object[] sortValues) {
// LUCENE 4 UPGRADE: There must be a better way
// we want to convert to a Text object here, and not BytesRef
// Don't write into sortValues! Otherwise the fields in FieldDoc is modified, which may be used in other places. (SearchContext#lastEmitedDoc)
Object[] sortValuesCopy = new Object[sortValues.length];
System.arraycopy(sortValues, 0, sortValuesCopy, 0, sortValues.length);
if (sortValues != null) {
for (int i = 0; i < sortValues.length; i++) {
if (sortValues[i] instanceof BytesRef) {
sortValuesCopy[i] = new Text(new BytesArray((BytesRef) sortValues[i]));
}
public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
for (int i = 0; i < sortValues.length; ++i) {
if (this.sortValues[i] instanceof BytesRef) {
this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
}
}
this.sortValues = sortValuesCopy;
}
@Override
@ -618,8 +611,6 @@ public class InternalSearchHit implements SearchHit {
sortValues[i] = in.readShort();
} else if (type == 8) {
sortValues[i] = in.readBoolean();
} else if (type == 9) {
sortValues[i] = in.readText();
} else {
throw new IOException("Can't match type [" + type + "]");
}
@ -726,9 +717,6 @@ public class InternalSearchHit implements SearchHit {
} else if (type == Boolean.class) {
out.writeByte((byte) 8);
out.writeBoolean((Boolean) sortValue);
} else if (sortValue instanceof Text) {
out.writeByte((byte) 9);
out.writeText((Text) sortValue);
} else {
throw new IOException("Can't handle sort field value of type [" + type + "]");
}

5
core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
View File

@ -59,6 +59,7 @@ import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.profile.Profilers;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import java.util.ArrayList;
@ -244,9 +245,9 @@ public abstract class SearchContext implements Releasable {
public abstract Float minimumScore();
public abstract SearchContext sort(Sort sort);
public abstract SearchContext sort(SortAndFormats sort);
public abstract Sort sort();
public abstract SortAndFormats sort();
public abstract SearchContext trackScores(boolean trackScores);

9
core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
View File

@ -19,19 +19,18 @@
package org.elasticsearch.search.internal;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Counter;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.search.aggregations.SearchContextAggregations;
import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
import org.elasticsearch.search.highlight.SearchContextHighlight;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import java.util.ArrayList;
@ -48,7 +47,7 @@ public class SubSearchContext extends FilteredSearchContext {
private int from;
private int size = DEFAULT_SIZE;
private Sort sort;
private SortAndFormats sort;
private ParsedQuery parsedQuery;
private Query query;
@ -172,13 +171,13 @@ public class SubSearchContext extends FilteredSearchContext {
}
@Override
public SearchContext sort(Sort sort) {
public SearchContext sort(SortAndFormats sort) {
this.sort = sort;
return this;
}
@Override
public Sort sort() {
public SortAndFormats sort() {
return sort;
}

19
core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
View File

@ -46,6 +46,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.MinimumScoreCollector;
import org.elasticsearch.common.lucene.search.FilteredCollector;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.SearchPhase;
import org.elasticsearch.search.SearchService;
@ -58,6 +59,7 @@ import org.elasticsearch.search.profile.ProfileShardResult;
import org.elasticsearch.search.profile.Profiler;
import org.elasticsearch.search.rescore.RescorePhase;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.sort.TrackScoresParseElement;
import org.elasticsearch.search.suggest.SuggestPhase;
@ -119,7 +121,9 @@ public class QueryPhase implements SearchPhase {
if (searchContext.hasOnlySuggest()) {
suggestPhase.execute(searchContext);
// TODO: fix this once we can fetch docs for suggestions
searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0));
searchContext.queryResult().topDocs(
new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0),
new DocValueFormat[0]);
return;
}
// Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F
@ -141,15 +145,15 @@ public class QueryPhase implements SearchPhase {
}
}
private static boolean returnsDocsInOrder(Query query, Sort sort) {
if (sort == null || Sort.RELEVANCE.equals(sort)) {
private static boolean returnsDocsInOrder(Query query, SortAndFormats sf) {
if (sf == null || Sort.RELEVANCE.equals(sf.sort)) {
// sort by score
// queries that return constant scores will return docs in index
// order since Lucene tie-breaks on the doc id
return query.getClass() == ConstantScoreQuery.class
|| query.getClass() == MatchAllDocsQuery.class;
} else {
return Sort.INDEXORDER.equals(sort);
return Sort.INDEXORDER.equals(sf.sort);
}
}
@ -176,6 +180,7 @@ public class QueryPhase implements SearchPhase {
Collector collector;
Callable<TopDocs> topDocsCallable;
DocValueFormat[] sortValueFormats = new DocValueFormat[0];
assert query == searcher.rewrite(query); // already rewritten
@ -229,8 +234,10 @@ public class QueryPhase implements SearchPhase {
}
assert numDocs > 0;
if (searchContext.sort() != null) {
topDocsCollector = TopFieldCollector.create(searchContext.sort(), numDocs,
SortAndFormats sf = searchContext.sort();
topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
(FieldDoc) after, true, searchContext.trackScores(), searchContext.trackScores());
sortValueFormats = sf.formats;
} else {
rescore = !searchContext.rescore().isEmpty();
for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
@ -402,7 +409,7 @@ public class QueryPhase implements SearchPhase {
queryResult.terminatedEarly(false);
}
queryResult.topDocs(topDocsCallable.call());
queryResult.topDocs(topDocsCallable.call(), sortValueFormats);
if (searchContext.getProfilers() != null) {
List<ProfileShardResult> shardResults = Profiler.buildShardResults(searchContext.getProfilers().getProfilers());

34
core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
View File

@ -19,12 +19,14 @@
package org.elasticsearch.search.query;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.TopDocs;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.InternalAggregations;
@ -51,6 +53,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
private int from;
private int size;
private TopDocs topDocs;
private DocValueFormat[] sortValueFormats;
private InternalAggregations aggregations;
private List<SiblingPipelineAggregator> pipelineAggregators;
private Suggest suggest;
@ -112,8 +115,20 @@ public class QuerySearchResult extends QuerySearchResultProvider {
return topDocs;
}
public void topDocs(TopDocs topDocs) {
public void topDocs(TopDocs topDocs, DocValueFormat[] sortValueFormats) {
this.topDocs = topDocs;
if (topDocs.scoreDocs.length > 0 && topDocs.scoreDocs[0] instanceof FieldDoc) {
int numFields = ((FieldDoc) topDocs.scoreDocs[0]).fields.length;
if (numFields != sortValueFormats.length) {
throw new IllegalArgumentException("The number of sort fields does not match: "
+ numFields + " != " + sortValueFormats.length);
}
}
this.sortValueFormats = sortValueFormats;
}
public DocValueFormat[] sortValueFormats() {
return sortValueFormats;
}
public Aggregations aggregations() {
@ -192,6 +207,15 @@ public class QuerySearchResult extends QuerySearchResultProvider {
// shardTarget = readSearchShardTarget(in);
from = in.readVInt();
size = in.readVInt();
int numSortFieldsPlus1 = in.readVInt();
if (numSortFieldsPlus1 == 0) {
sortValueFormats = null;
} else {
sortValueFormats = new DocValueFormat[numSortFieldsPlus1 - 1];
for (int i = 0; i < sortValueFormats.length; ++i) {
sortValueFormats[i] = in.readNamedWriteable(DocValueFormat.class);
}
}
topDocs = readTopDocs(in);
if (in.readBoolean()) {
aggregations = InternalAggregations.readAggregations(in);
@ -233,6 +257,14 @@ public class QuerySearchResult extends QuerySearchResultProvider {
// shardTarget.writeTo(out);
out.writeVInt(from);
out.writeVInt(size);
if (sortValueFormats == null) {
out.writeVInt(0);
} else {
out.writeVInt(1 + sortValueFormats.length);
for (int i = 0; i < sortValueFormats.length; ++i) {
out.writeNamedWriteable(sortValueFormats[i]);
}
}
writeTopDocs(out, topDocs);
if (aggregations == null) {
out.writeBoolean(false);

2
core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
View File

@ -61,7 +61,7 @@ public class RescorePhase extends AbstractComponent implements SearchPhase {
for (RescoreSearchContext ctx : context.rescore()) {
topDocs = ctx.rescorer().rescore(topDocs, context, ctx);
}
context.queryResult().topDocs(topDocs);
context.queryResult().topDocs(topDocs, context.queryResult().sortValueFormats());
} catch (IOException e) {
throw new ElasticsearchException("Rescore Phase Failed", e);
}

26
core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
View File

@ -20,9 +20,7 @@
package org.elasticsearch.search.searchafter;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
@ -36,6 +34,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.sort.SortAndFormats;
import java.io.IOException;
import java.util.ArrayList;
@ -104,21 +104,23 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
return Arrays.copyOf(sortValues, sortValues.length);
}
public static FieldDoc buildFieldDoc(Sort sort, Object[] values) {
if (sort == null || sort.getSort() == null || sort.getSort().length == 0) {
public static FieldDoc buildFieldDoc(SortAndFormats sort, Object[] values) {
if (sort == null || sort.sort.getSort() == null || sort.sort.getSort().length == 0) {
throw new IllegalArgumentException("Sort must contain at least one field.");
}
SortField[] sortFields = sort.getSort();
SortField[] sortFields = sort.sort.getSort();
if (sortFields.length != values.length) {
throw new IllegalArgumentException(
SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has " + sort.getSort().length + ".");
SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has "
+ sort.sort.getSort().length + ".");
}
Object[] fieldValues = new Object[sortFields.length];
for (int i = 0; i < sortFields.length; i++) {
SortField sortField = sortFields[i];
DocValueFormat format = sort.formats[i];
if (values[i] != null) {
fieldValues[i] = convertValueFromSortField(values[i], sortField);
fieldValues[i] = convertValueFromSortField(values[i], sortField, format);
} else {
fieldValues[i] = null;
}
@ -130,15 +132,15 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
return new FieldDoc(Integer.MAX_VALUE, 0, fieldValues);
}
private static Object convertValueFromSortField(Object value, SortField sortField) {
private static Object convertValueFromSortField(Object value, SortField sortField, DocValueFormat format) {
if (sortField.getComparatorSource() instanceof IndexFieldData.XFieldComparatorSource) {
IndexFieldData.XFieldComparatorSource cmpSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value);
return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value, format);
}
return convertValueFromSortType(sortField.getField(), sortField.getType(), value);
return convertValueFromSortType(sortField.getField(), sortField.getType(), value, format);
}
private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value) {
private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value, DocValueFormat format) {
try {
switch (sortType) {
case DOC:
@ -179,7 +181,7 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
case STRING_VAL:
case STRING:
return new BytesRef(value.toString());
return format.parseBytesRef(value.toString());
default:
throw new IllegalArgumentException("Comparator type [" + sortType.name() + "] for field [" + fieldName

12
core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -55,8 +56,10 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
* special field name to sort by index order
*/
public static final String DOC_FIELD_NAME = "_doc";
private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC);
private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true);
private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat(
new SortField(null, SortField.Type.DOC), DocValueFormat.RAW);
private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat(
new SortField(null, SortField.Type.DOC, true), DocValueFormat.RAW);
private final String fieldName;
@ -246,7 +249,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
}
@Override
public SortField build(QueryShardContext context) throws IOException {
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
if (DOC_FIELD_NAME.equals(fieldName)) {
if (order == SortOrder.DESC) {
return SORT_DOC_REVERSE;
@ -281,7 +284,8 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
}
IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData
.comparatorSource(missing, localSortMode, nested);
return new SortField(fieldType.name(), fieldComparatorSource, reverse);
SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse);
return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
}
}

5
core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
View File

@ -51,6 +51,7 @@ import org.elasticsearch.index.query.GeoValidationMethod;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -504,7 +505,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
}
@Override
public SortField build(QueryShardContext context) throws IOException {
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
// validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
List<GeoPoint> localPoints = new ArrayList<GeoPoint>();
@ -585,7 +586,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
};
return new SortField(fieldName, geoDistanceComparatorSource, reverse);
return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW);
}
static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {

9
core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import java.io.IOException;
import java.util.Objects;
@ -40,8 +41,10 @@ public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
public static final String NAME = "_score";
public static final ParseField ORDER_FIELD = new ParseField("order");
private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE);
private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true);
private static final SortFieldAndFormat SORT_SCORE = new SortFieldAndFormat(
new SortField(null, SortField.Type.SCORE), DocValueFormat.RAW);
private static final SortFieldAndFormat SORT_SCORE_REVERSE = new SortFieldAndFormat(
new SortField(null, SortField.Type.SCORE, true), DocValueFormat.RAW);
/**
* Build a ScoreSortBuilder default to descending sort order.
@ -106,7 +109,7 @@ public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
}
@Override
public SortField build(QueryShardContext context) {
public SortFieldAndFormat build(QueryShardContext context) {
if (order == SortOrder.DESC) {
return SORT_SCORE;
} else {

5
core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
View File

@ -52,6 +52,7 @@ import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptParameterParser;
import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -302,7 +303,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
@Override
public SortField build(QueryShardContext context) throws IOException {
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
final SearchScript searchScript = context.getScriptService().search(
context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), context.getClusterState());
@ -366,7 +367,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
throw new QueryShardException(context, "custom script sort type [" + type + "] not supported");
}
return new SortField("_script", fieldComparatorSource, reverse);
return new SortFieldAndFormat(new SortField("_script", fieldComparatorSource, reverse), DocValueFormat.RAW);
}
@Override

38
core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.sort;
import org.apache.lucene.search.Sort;
import org.elasticsearch.search.DocValueFormat;
public final class SortAndFormats {
public final Sort sort;
public final DocValueFormat[] formats;
public SortAndFormats(Sort sort, DocValueFormat[] formats) {
if (sort.getSort().length != formats.length) {
throw new IllegalArgumentException("Number of sort field mismatch: "
+ sort.getSort().length + " != " + formats.length);
}
this.sort = sort;
this.formats = formats;
}
}

16
core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import java.io.IOException;
import java.util.ArrayList;
@ -65,9 +66,9 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
}
/**
* Create a @link {@link SortField} from this builder.
* Create a @link {@link SortFieldAndFormat} from this builder.
*/
protected abstract SortField build(QueryShardContext context) throws IOException;
protected abstract SortFieldAndFormat build(QueryShardContext context) throws IOException;
/**
* Set the order of sorting.
@ -143,10 +144,13 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
}
}
public static Optional<Sort> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
public static Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
List<SortField> sortFields = new ArrayList<>(sortBuilders.size());
List<DocValueFormat> sortFormats = new ArrayList<>(sortBuilders.size());
for (SortBuilder<?> builder : sortBuilders) {
sortFields.add(builder.build(context));
SortFieldAndFormat sf = builder.build(context);
sortFields.add(sf.field);
sortFormats.add(sf.format);
}
if (!sortFields.isEmpty()) {
// optimize if we just sort on score non reversed, we don't really
@ -163,7 +167,9 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
}
}
if (sort) {
return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()])));
return Optional.of(new SortAndFormats(
new Sort(sortFields.toArray(new SortField[sortFields.size()])),
sortFormats.toArray(new DocValueFormat[sortFormats.size()])));
}
}
return Optional.empty();

36
core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.sort;
import java.util.Objects;
import org.apache.lucene.search.SortField;
import org.elasticsearch.search.DocValueFormat;
public final class SortFieldAndFormat {
public final SortField field;
public final DocValueFormat format;
public SortFieldAndFormat(SortField field, DocValueFormat format) {
this.field = Objects.requireNonNull(field);
this.format = Objects.requireNonNull(format);
}
}

1
core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java
View File

@ -100,4 +100,5 @@ public class NetworkAddressTests extends ESTestCase {
byte bytes[] = InetAddress.getByName(address).getAddress();
return Inet6Address.getByAddress(hostname, bytes, scopeid);
}
}

4
core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
View File

@ -140,8 +140,8 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
InnerHitsContext.BaseInnerHits innerHits =
searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
assertEquals(innerHits.sort().getSort().length, 1);
assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
assertEquals(innerHits.sort().sort.getSort().length, 1);
assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
} else {
assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
}

4
core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
View File

@ -121,8 +121,8 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits()
.getInnerHits().get(queryBuilder.innerHit().getName());
assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
assertEquals(innerHits.sort().getSort().length, 1);
assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
assertEquals(innerHits.sort().sort.getSort().length, 1);
assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
} else {
assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
}

4
core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
View File

@ -105,8 +105,8 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
assertEquals(innerHits.sort().getSort().length, 1);
assertEquals(innerHits.sort().getSort()[0].getField(), INT_FIELD_NAME);
assertEquals(innerHits.sort().sort.getSort().length, 1);
assertEquals(innerHits.sort().sort.getSort()[0].getField(), INT_FIELD_NAME);
} else {
assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
}

64
core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
View File

@ -19,11 +19,14 @@
package org.elasticsearch.search;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTimeZone;
@ -76,4 +79,65 @@ public class DocValueFormatTests extends ESTestCase {
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
}
public void testRawFormat() {
assertEquals("0", DocValueFormat.RAW.format(0));
assertEquals("-1", DocValueFormat.RAW.format(-1));
assertEquals("1", DocValueFormat.RAW.format(1));
assertEquals("0.0", DocValueFormat.RAW.format(0d));
assertEquals("0.5", DocValueFormat.RAW.format(.5d));
assertEquals("-1.0", DocValueFormat.RAW.format(-1d));
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
}
public void testBooleanFormat() {
assertEquals("false", DocValueFormat.BOOLEAN.format(0));
assertEquals("true", DocValueFormat.BOOLEAN.format(1));
}
public void testIpFormat() {
assertEquals("192.168.1.7",
DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.7")))));
assertEquals("::1",
DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1")))));
}
public void testRawParse() {
assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null));
assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null));
// not checking exception messages as they could depend on the JVM
expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("", randomBoolean(), null));
expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null));
assertEquals(-1d, DocValueFormat.RAW.parseDouble("-1", randomBoolean(), null), 0d);
assertEquals(1d, DocValueFormat.RAW.parseDouble("1", randomBoolean(), null), 0d);
assertEquals(.5, DocValueFormat.RAW.parseDouble("0.5", randomBoolean(), null), 0d);
// not checking exception messages as they could depend on the JVM
expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("", randomBoolean(), null));
expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null));
assertEquals(new BytesRef("abc"), DocValueFormat.RAW.parseBytesRef("abc"));
}
public void testBooleanParse() {
assertEquals(0L, DocValueFormat.BOOLEAN.parseLong("false", randomBoolean(), null));
assertEquals(1L, DocValueFormat.BOOLEAN.parseLong("true", randomBoolean(), null));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> DocValueFormat.BOOLEAN.parseLong("", randomBoolean(), null));
assertEquals("Cannot parse boolean [], expected either [true] or [false]", e.getMessage());
e = expectThrows(IllegalArgumentException.class,
() -> DocValueFormat.BOOLEAN.parseLong("0", randomBoolean(), null));
assertEquals("Cannot parse boolean [0], expected either [true] or [false]", e.getMessage());
e = expectThrows(IllegalArgumentException.class,
() -> DocValueFormat.BOOLEAN.parseLong("False", randomBoolean(), null));
assertEquals("Cannot parse boolean [False], expected either [true] or [false]", e.getMessage());
}
public void testIPParse() {
assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.7"))),
DocValueFormat.IP.parseBytesRef("192.168.1.7"));
assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
DocValueFormat.IP.parseBytesRef("::1"));
}
}

7
core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.SearchContextException;
import org.elasticsearch.search.SearchHit;
@ -189,11 +188,11 @@ public class SearchAfterIT extends ESIntegTestCase {
values.add(randomDouble());
break;
case 6:
values.add(new Text(randomAsciiOfLengthBetween(5, 20)));
values.add(randomAsciiOfLengthBetween(5, 20));
break;
}
}
values.add(new Text(UUIDs.randomBase64UUID()));
values.add(UUIDs.randomBase64UUID());
documents.add(values);
}
int reqSize = randomInt(NUM_DOCS-1);
@ -296,7 +295,7 @@ public class SearchAfterIT extends ESIntegTestCase {
} else if (type == Boolean.class) {
mappings.add("field" + Integer.toString(i));
mappings.add("type=boolean");
} else if (types.get(i) instanceof Text) {
} else if (types.get(i) instanceof String) {
mappings.add("field" + Integer.toString(i));
mappings.add("type=keyword");
} else {

7
core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
View File

@ -62,6 +62,7 @@ import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -163,12 +164,12 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
QueryShardContext mockShardContext = createMockShardContext();
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
T sortBuilder = createTestItem();
SortField sortField = sortBuilder.build(mockShardContext);
sortFieldAssertions(sortBuilder, sortField);
SortFieldAndFormat sortField = sortBuilder.build(mockShardContext);
sortFieldAssertions(sortBuilder, sortField.field, sortField.format);
}
}
protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException;
protected abstract void sortFieldAssertions(T builder, SortField sortField, DocValueFormat format) throws IOException;
/**
* Test serialization and deserialization of the test sort.

4
core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.DocValueFormat;
import java.io.IOException;
import java.util.Arrays;
@ -110,7 +111,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
}
@Override
protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField) throws IOException {
protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
SortField.Type expectedType;
if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) {
expectedType = SortField.Type.DOC;
@ -122,6 +123,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
if (expectedType == SortField.Type.CUSTOM) {
assertEquals(builder.getFieldName(), sortField.getField());
}
assertEquals(DocValueFormat.RAW, format);
}
public void testReverseOptionFails() throws IOException {

72
core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
View File

@ -29,7 +29,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.Uid;
@ -76,7 +75,6 @@ import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class FieldSortIT extends ESIntegTestCase {
@ -1214,13 +1212,13 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("!4"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("!4"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("01"));
assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("01"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("07"));
assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("07"));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
@ -1232,13 +1230,13 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
}
public void testSortOnRareField() throws IOException {
@ -1262,7 +1260,7 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("10"));
client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject()
.array("string_values", "11", "15", "20", "07")
@ -1283,10 +1281,10 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().hits().length, equalTo(2));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject()
@ -1308,13 +1306,13 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
for (int i = 0; i < 15; i++) {
client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject()
@ -1332,13 +1330,13 @@ public class FieldSortIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
}
public void testSortMetaField() throws Exception {
@ -1448,10 +1446,7 @@ public class FieldSortIT extends ESIntegTestCase {
SearchHit[] hits = searchResponse.getHits().hits();
for (int i = 0; i < hits.length; ++i) {
assertThat(hits[i].getSortValues().length, is(1));
Object o = hits[i].getSortValues()[0];
assertThat(o, notNullValue());
Text text = (Text) o;
assertThat(text.string(), is("bar"));
assertThat(hits[i].getSortValues()[0], is("bar"));
}
@ -1464,10 +1459,7 @@ public class FieldSortIT extends ESIntegTestCase {
hits = searchResponse.getHits().hits();
for (int i = 0; i < hits.length; ++i) {
assertThat(hits[i].getSortValues().length, is(1));
Object o = hits[i].getSortValues()[0];
assertThat(o, notNullValue());
Text text = (Text) o;
assertThat(text.string(), is("bar bar"));
assertThat(hits[i].getSortValues()[0], is("bar bar"));
}
}
@ -1506,4 +1498,34 @@ public class FieldSortIT extends ESIntegTestCase {
}
}
public void testCustomFormat() throws Exception {
// Use an ip field, which uses different internal/external
// representations of values, to make sure values are both correctly
// rendered and parsed (search_after)
assertAcked(prepareCreate("test")
.addMapping("type", "ip", "type=ip"));
indexRandom(true,
client().prepareIndex("test", "type", "1").setSource("ip", "192.168.1.7"),
client().prepareIndex("test", "type", "2").setSource("ip", "2001:db8::ff00:42:8329"));
SearchResponse response = client().prepareSearch("test")
.addSort(SortBuilders.fieldSort("ip"))
.get();
assertSearchResponse(response);
assertEquals(2, response.getHits().totalHits());
assertArrayEquals(new String[] {"192.168.1.7"},
response.getHits().getAt(0).getSortValues());
assertArrayEquals(new String[] {"2001:db8::ff00:42:8329"},
response.getHits().getAt(1).getSortValues());
response = client().prepareSearch("test")
.addSort(SortBuilders.fieldSort("ip"))
.searchAfter(new Object[] {"192.168.1.7"})
.get();
assertSearchResponse(response);
assertEquals(2, response.getHits().totalHits());
assertEquals(1, response.getHits().hits().length);
assertArrayEquals(new String[] {"2001:db8::ff00:42:8329"},
response.getHits().getAt(0).getSortValues());
}
}

2
core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
View File

@ -350,7 +350,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
.addSort(fieldSort("str_field2").order(SortOrder.DESC).unmappedType("keyword")).get();
assertSortValues(resp,
new Object[] {new Text("bcd"), null},
new Object[] {"bcd", null},
new Object[] {null, null});
resp = client().prepareSearch("test1", "test2")

3
core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
import org.elasticsearch.index.query.GeoValidationMethod;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.test.geo.RandomGeoGenerator;
import java.io.IOException;
@ -177,7 +178,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
}
@Override
protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField) throws IOException {
protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
assertEquals(SortField.Type.CUSTOM, sortField.getType());
assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
assertEquals(builder.fieldName(), sortField.getField());

3
core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.DocValueFormat;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
@ -98,7 +99,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
}
@Override
protected void sortFieldAssertions(ScoreSortBuilder builder, SortField sortField) {
protected void sortFieldAssertions(ScoreSortBuilder builder, SortField sortField, DocValueFormat format) {
assertEquals(SortField.Type.SCORE, sortField.getType());
assertEquals(builder.order() == SortOrder.DESC ? false : true, sortField.getReverse());
}

3
core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService.ScriptType;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
@ -124,7 +125,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
}
@Override
protected void sortFieldAssertions(ScriptSortBuilder builder, SortField sortField) throws IOException {
protected void sortFieldAssertions(ScriptSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
assertEquals(SortField.Type.CUSTOM, sortField.getType());
assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
}

4
docs/reference/migration/migrate_5_0/java.asciidoc
View File

@ -270,6 +270,10 @@ at call time which results in much clearer errors.
All `extraSource` methods have been removed.
==== SearchResponse
Sort values for `string` fields are now return as `java.lang.String` objects rather than `org.elasticsearch.common.text.Text`.
==== AggregationBuilder
All methods which take an `XContentBuilder`, `BytesReference` `Map<String, Object>` or `bytes[]` have been removed in favor of providing the

12
test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
View File

@ -18,16 +18,13 @@
*/
package org.elasticsearch.test;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Counter;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cache.recycler.PageCacheRecycler;
@ -65,13 +62,10 @@ import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.profile.Profilers;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.search.rescore.RescoreSearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class TestSearchContext extends SearchContext {
final PageCacheRecycler pageCacheRecycler;
@ -365,12 +359,12 @@ public class TestSearchContext extends SearchContext {
}
@Override
public SearchContext sort(Sort sort) {
public SearchContext sort(SortAndFormats sort) {
return null;
}
@Override
public Sort sort() {
public SortAndFormats sort() {
return null;
}