From de8354dd7fa93dbcc71f5dbf154079f92b2b0bf0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 25 Apr 2016 17:45:33 +0200 Subject: [PATCH] Allow binary sort values. #17959 The `ip` field uses a binary representation internally. This breaks when rendering sort values in search responses since elasticsearch tries to write a binary byte[] as an utf8 json string. This commit extends the `DocValueFormat` API in order to give fields a chance to choose how to render values. Closes #6077 --- .../index/query/InnerHitBuilder.java | 4 +- .../elasticsearch/search/DocValueFormat.java | 58 +++++++++++++-- .../elasticsearch/search/SearchService.java | 3 +- .../metrics/tophits/TopHitsAggregator.java | 16 ++--- .../tophits/TopHitsAggregatorFactory.java | 4 +- .../controller/SearchPhaseController.java | 2 +- .../fetch/innerhits/InnerHitsContext.java | 4 +- .../innerhits/InnerHitsFetchSubPhase.java | 4 +- .../search/internal/DefaultSearchContext.java | 8 +-- .../internal/FilteredSearchContext.java | 6 +- .../search/internal/InternalSearchHit.java | 26 ++----- .../search/internal/SearchContext.java | 5 +- .../search/internal/SubSearchContext.java | 9 ++- .../search/query/QueryPhase.java | 19 +++-- .../search/query/QuerySearchResult.java | 34 ++++++++- .../search/rescore/RescorePhase.java | 2 +- .../searchafter/SearchAfterBuilder.java | 26 +++---- .../search/sort/FieldSortBuilder.java | 12 ++-- .../search/sort/GeoDistanceSortBuilder.java | 5 +- .../search/sort/ScoreSortBuilder.java | 9 ++- .../search/sort/ScriptSortBuilder.java | 5 +- .../search/sort/SortAndFormats.java | 38 ++++++++++ .../search/sort/SortBuilder.java | 16 +++-- .../search/sort/SortFieldAndFormat.java | 36 ++++++++++ .../common/network/NetworkAddressTests.java | 1 + .../query/HasChildQueryBuilderTests.java | 4 +- .../query/HasParentQueryBuilderTests.java | 4 +- .../index/query/NestedQueryBuilderTests.java | 4 +- .../search/DocValueFormatTests.java | 64 +++++++++++++++++ .../search/searchafter/SearchAfterIT.java | 7 +- .../search/sort/AbstractSortTestCase.java | 7 +- .../search/sort/FieldSortBuilderTests.java | 4 +- .../search/sort/FieldSortIT.java | 72 ++++++++++++------- .../search/sort/GeoDistanceSortBuilderIT.java | 2 +- .../sort/GeoDistanceSortBuilderTests.java | 3 +- .../search/sort/ScoreSortBuilderTests.java | 3 +- .../search/sort/ScriptSortBuilderTests.java | 3 +- .../migration/migrate_5_0/java.asciidoc | 4 ++ .../elasticsearch/test/TestSearchContext.java | 12 +--- 39 files changed, 402 insertions(+), 143 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java create mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index c07c75532f1..b62b5a18a1d 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.query; -import org.apache.lucene.search.Sort; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -41,6 +40,7 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import java.io.IOException; @@ -512,7 +512,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl innerHitsContext.fetchSourceContext(fetchSourceContext); } if (sorts != null) { - Optional optionalSort = SortBuilder.buildSort(sorts, context); + Optional optionalSort = SortBuilder.buildSort(sorts, context); if (optionalSort.isPresent()) { innerHitsContext.sort(optionalSort.get()); } diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java index 78f8460f1cc..c7c993b82b6 100644 --- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -20,7 +20,6 @@ package org.elasticsearch.search; import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.index.Term; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.io.stream.NamedWriteable; @@ -29,8 +28,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.joda.DateMathParser; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; -import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper; import org.joda.time.DateTimeZone; @@ -48,16 +47,33 @@ import java.util.concurrent.Callable; /** A formatter for values as returned by the fielddata/doc-values APIs. */ public interface DocValueFormat extends NamedWriteable { + /** Format a long value. This is used by terms and histogram aggregations + * to format keys for fields that use longs as a doc value representation + * such as the {@code long} and {@code date} fields. */ String format(long value); + /** Format a double value. This is used by terms and stats aggregations + * to format keys for fields that use numbers as a doc value representation + * such as the {@code long}, {@code double} or {@code date} fields. */ String format(double value); + /** Format a double value. This is used by terms aggregations to format + * keys for fields that use binary doc value representations such as the + * {@code keyword} and {@code ip} fields. */ String format(BytesRef value); + /** Parse a value that was formatted with {@link #format(long)} back to the + * original long value. */ long parseLong(String value, boolean roundUp, Callable now); + /** Parse a value that was formatted with {@link #format(double)} back to + * the original double value. */ double parseDouble(String value, boolean roundUp, Callable now); + /** Parse a value that was formatted with {@link #format(BytesRef)} back + * to the original BytesRef. */ + BytesRef parseBytesRef(String value); + public static final DocValueFormat RAW = new DocValueFormat() { @Override @@ -81,7 +97,7 @@ public interface DocValueFormat extends NamedWriteable { @Override public String format(BytesRef value) { - return Term.toString(value); + return value.utf8ToString(); } @Override @@ -99,6 +115,10 @@ public interface DocValueFormat extends NamedWriteable { public double parseDouble(String value, boolean roundUp, Callable now) { return Double.parseDouble(value); } + + public BytesRef parseBytesRef(String value) { + return new BytesRef(value); + } }; public static final class DateTime implements DocValueFormat { @@ -154,6 +174,11 @@ public interface DocValueFormat extends NamedWriteable { public double parseDouble(String value, boolean roundUp, Callable now) { return parseLong(value, roundUp, now); } + + @Override + public BytesRef parseBytesRef(String value) { + throw new UnsupportedOperationException(); + } } public static final DocValueFormat GEOHASH = new DocValueFormat() { @@ -191,6 +216,11 @@ public interface DocValueFormat extends NamedWriteable { public double parseDouble(String value, boolean roundUp, Callable now) { throw new UnsupportedOperationException(); } + + @Override + public BytesRef parseBytesRef(String value) { + throw new UnsupportedOperationException(); + } }; public static final DocValueFormat BOOLEAN = new DocValueFormat() { @@ -221,13 +251,24 @@ public interface DocValueFormat extends NamedWriteable { @Override public long parseLong(String value, boolean roundUp, Callable now) { - throw new UnsupportedOperationException(); + switch (value) { + case "false": + return 0; + case "true": + return 1; + } + throw new IllegalArgumentException("Cannot parse boolean [" + value + "], expected either [true] or [false]"); } @Override public double parseDouble(String value, boolean roundUp, Callable now) { throw new UnsupportedOperationException(); } + + @Override + public BytesRef parseBytesRef(String value) { + throw new UnsupportedOperationException(); + } }; public static final DocValueFormat IP = new DocValueFormat() { @@ -268,6 +309,11 @@ public interface DocValueFormat extends NamedWriteable { public double parseDouble(String value, boolean roundUp, Callable now) { return parseLong(value, roundUp, now); } + + @Override + public BytesRef parseBytesRef(String value) { + return new BytesRef(InetAddressPoint.encode(InetAddresses.forString(value))); + } }; public static final class Decimal implements DocValueFormat { @@ -344,5 +390,9 @@ public interface DocValueFormat extends NamedWriteable { return n.doubleValue(); } + @Override + public BytesRef parseBytesRef(String value) { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 24746431949..636e4338063 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -104,6 +104,7 @@ import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.search.query.ScrollQuerySearchResult; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.threadpool.ThreadPool; @@ -698,7 +699,7 @@ public class SearchService extends AbstractLifecycleComponent imp } if (source.sorts() != null) { try { - Optional optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext()); + Optional optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext()); if (optionalSort.isPresent()) { context.sort(optionalSort.get()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java index 8e50dc60ea8..07292f1d29f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java @@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.search.TopFieldCollector; @@ -45,6 +44,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import java.io.IOException; import java.util.List; @@ -78,9 +78,9 @@ public class TopHitsAggregator extends MetricsAggregator { @Override public boolean needsScores() { - Sort sort = subSearchContext.sort(); + SortAndFormats sort = subSearchContext.sort(); if (sort != null) { - return sort.needsScores() || subSearchContext.trackScores(); + return sort.sort.needsScores() || subSearchContext.trackScores(); } else { // sort by score return true; @@ -112,12 +112,12 @@ public class TopHitsAggregator extends MetricsAggregator { public void collect(int docId, long bucket) throws IOException { TopDocsAndLeafCollector collectors = topDocsCollectors.get(bucket); if (collectors == null) { - Sort sort = subSearchContext.sort(); + SortAndFormats sort = subSearchContext.sort(); int topN = subSearchContext.from() + subSearchContext.size(); // In the QueryPhase we don't need this protection, because it is build into the IndexSearcher, // but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size. topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc()); - TopDocsCollector topLevelCollector = sort != null ? TopFieldCollector.create(sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN); + TopDocsCollector topLevelCollector = sort != null ? TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN); collectors = new TopDocsAndLeafCollector(topLevelCollector); collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx); collectors.leafCollector.setScorer(scorer); @@ -137,7 +137,7 @@ public class TopHitsAggregator extends MetricsAggregator { } else { final TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs(); - subSearchContext.queryResult().topDocs(topDocs); + subSearchContext.queryResult().topDocs(topDocs, subSearchContext.sort() == null ? null : subSearchContext.sort().formats); int[] docIdsToLoad = new int[topDocs.scoreDocs.length]; for (int i = 0; i < topDocs.scoreDocs.length; i++) { docIdsToLoad[i] = topDocs.scoreDocs[i].doc; @@ -153,7 +153,7 @@ public class TopHitsAggregator extends MetricsAggregator { searchHitFields.score(scoreDoc.score); if (scoreDoc instanceof FieldDoc) { FieldDoc fieldDoc = (FieldDoc) scoreDoc; - searchHitFields.sortValues(fieldDoc.fields); + searchHitFields.sortValues(fieldDoc.fields, subSearchContext.sort().formats); } } topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(), @@ -166,7 +166,7 @@ public class TopHitsAggregator extends MetricsAggregator { public InternalTopHits buildEmptyAggregation() { TopDocs topDocs; if (subSearchContext.sort() != null) { - topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().getSort(), Float.NaN); + topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().sort.getSort(), Float.NaN); } else { topDocs = Lucene.EMPTY_TOP_DOCS; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index 478f03c7eac..ac001222301 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits; -import org.apache.lucene.search.Sort; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.Aggregator; @@ -35,6 +34,7 @@ import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import java.io.IOException; @@ -87,7 +87,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext()); + Optional optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext()); if (optionalSort.isPresent()) { subSearchContext.sort(optionalSort.get()); } diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java index 1c48be1b959..a8ecc7a508c 100644 --- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java +++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java @@ -362,7 +362,7 @@ public class SearchPhaseController extends AbstractComponent { if (sorted) { FieldDoc fieldDoc = (FieldDoc) shardDoc; - searchHit.sortValues(fieldDoc.fields); + searchHit.sortValues(fieldDoc.fields, firstResult.sortValueFormats()); if (sortScoreIndex != -1) { searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue()); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java index 56059fc0d01..31921457207 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java @@ -142,7 +142,7 @@ public final class InnerHitsContext { TopDocsCollector topDocsCollector; if (sort() != null) { try { - topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores()); + topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); } @@ -317,7 +317,7 @@ public final class InnerHitsContext { int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); TopDocsCollector topDocsCollector; if (sort() != null) { - topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores()); + topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); } else { topDocsCollector = TopScoreDocCollector.create(topN); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java index ea7edcc3dd4..82c3755cdc9 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java @@ -73,7 +73,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase { } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); } - innerHits.queryResult().topDocs(topDocs); + innerHits.queryResult().topDocs(topDocs, innerHits.sort() == null ? null : innerHits.sort().formats); int[] docIdsToLoad = new int[topDocs.scoreDocs.length]; for (int i = 0; i < topDocs.scoreDocs.length; i++) { docIdsToLoad[i] = topDocs.scoreDocs[i].doc; @@ -89,7 +89,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase { searchHitFields.score(scoreDoc.score); if (scoreDoc instanceof FieldDoc) { FieldDoc fieldDoc = (FieldDoc) scoreDoc; - searchHitFields.sortValues(fieldDoc.fields); + searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats); } } results.put(entry.getKey(), fetchResult.hits()); diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 435e809a893..96319303420 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -71,6 +70,7 @@ import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QueryPhaseExecutionException; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; import java.io.IOException; @@ -114,7 +114,7 @@ public class DefaultSearchContext extends SearchContext { private FetchSourceContext fetchSourceContext; private int from = -1; private int size = -1; - private Sort sort; + private SortAndFormats sort; private Float minimumScore; private boolean trackScores = false; // when sorting, track scores as well... private FieldDoc searchAfter; @@ -532,13 +532,13 @@ public class DefaultSearchContext extends SearchContext { } @Override - public SearchContext sort(Sort sort) { + public SearchContext sort(SortAndFormats sort) { this.sort = sort; return this; } @Override - public Sort sort() { + public SortAndFormats sort() { return this.sort; } diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 283e1dd60e8..8009d0b5fe4 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.internal; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -55,6 +54,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; import java.util.List; @@ -306,12 +306,12 @@ public abstract class FilteredSearchContext extends SearchContext { } @Override - public SearchContext sort(Sort sort) { + public SearchContext sort(SortAndFormats sort) { return in.sort(sort); } @Override - public Sort sort() { + public SortAndFormats sort() { return in.sort(); } diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index 628f409974e..f67ed51b226 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -24,7 +24,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,6 +33,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; @@ -44,6 +44,7 @@ import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -326,21 +327,13 @@ public class InternalSearchHit implements SearchHit { this.highlightFields = highlightFields; } - public void sortValues(Object[] sortValues) { - // LUCENE 4 UPGRADE: There must be a better way - // we want to convert to a Text object here, and not BytesRef - - // Don't write into sortValues! Otherwise the fields in FieldDoc is modified, which may be used in other places. (SearchContext#lastEmitedDoc) - Object[] sortValuesCopy = new Object[sortValues.length]; - System.arraycopy(sortValues, 0, sortValuesCopy, 0, sortValues.length); - if (sortValues != null) { - for (int i = 0; i < sortValues.length; i++) { - if (sortValues[i] instanceof BytesRef) { - sortValuesCopy[i] = new Text(new BytesArray((BytesRef) sortValues[i])); - } + public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) { + this.sortValues = Arrays.copyOf(sortValues, sortValues.length); + for (int i = 0; i < sortValues.length; ++i) { + if (this.sortValues[i] instanceof BytesRef) { + this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]); } } - this.sortValues = sortValuesCopy; } @Override @@ -618,8 +611,6 @@ public class InternalSearchHit implements SearchHit { sortValues[i] = in.readShort(); } else if (type == 8) { sortValues[i] = in.readBoolean(); - } else if (type == 9) { - sortValues[i] = in.readText(); } else { throw new IOException("Can't match type [" + type + "]"); } @@ -726,9 +717,6 @@ public class InternalSearchHit implements SearchHit { } else if (type == Boolean.class) { out.writeByte((byte) 8); out.writeBoolean((Boolean) sortValue); - } else if (sortValue instanceof Text) { - out.writeByte((byte) 9); - out.writeText((Text) sortValue); } else { throw new IOException("Can't handle sort field value of type [" + type + "]"); } diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 8b55c764bbb..550a5f76caf 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -59,6 +59,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; import java.util.ArrayList; @@ -244,9 +245,9 @@ public abstract class SearchContext implements Releasable { public abstract Float minimumScore(); - public abstract SearchContext sort(Sort sort); + public abstract SearchContext sort(SortAndFormats sort); - public abstract Sort sort(); + public abstract SortAndFormats sort(); public abstract SearchContext trackScores(boolean trackScores); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index be2c3798430..6cacf86d65f 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -19,19 +19,18 @@ package org.elasticsearch.search.internal; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.fetch.FetchSearchResult; -import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.SearchContextHighlight; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; import java.util.ArrayList; @@ -48,7 +47,7 @@ public class SubSearchContext extends FilteredSearchContext { private int from; private int size = DEFAULT_SIZE; - private Sort sort; + private SortAndFormats sort; private ParsedQuery parsedQuery; private Query query; @@ -172,13 +171,13 @@ public class SubSearchContext extends FilteredSearchContext { } @Override - public SearchContext sort(Sort sort) { + public SearchContext sort(SortAndFormats sort) { this.sort = sort; return this; } @Override - public Sort sort() { + public SortAndFormats sort() { return sort; } diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 62210655a00..a29831f5a04 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.MinimumScoreCollector; import org.elasticsearch.common.lucene.search.FilteredCollector; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; @@ -58,6 +59,7 @@ import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.Profiler; import org.elasticsearch.search.rescore.RescorePhase; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.TrackScoresParseElement; import org.elasticsearch.search.suggest.SuggestPhase; @@ -119,7 +121,9 @@ public class QueryPhase implements SearchPhase { if (searchContext.hasOnlySuggest()) { suggestPhase.execute(searchContext); // TODO: fix this once we can fetch docs for suggestions - searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0)); + searchContext.queryResult().topDocs( + new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0), + new DocValueFormat[0]); return; } // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F @@ -141,15 +145,15 @@ public class QueryPhase implements SearchPhase { } } - private static boolean returnsDocsInOrder(Query query, Sort sort) { - if (sort == null || Sort.RELEVANCE.equals(sort)) { + private static boolean returnsDocsInOrder(Query query, SortAndFormats sf) { + if (sf == null || Sort.RELEVANCE.equals(sf.sort)) { // sort by score // queries that return constant scores will return docs in index // order since Lucene tie-breaks on the doc id return query.getClass() == ConstantScoreQuery.class || query.getClass() == MatchAllDocsQuery.class; } else { - return Sort.INDEXORDER.equals(sort); + return Sort.INDEXORDER.equals(sf.sort); } } @@ -176,6 +180,7 @@ public class QueryPhase implements SearchPhase { Collector collector; Callable topDocsCallable; + DocValueFormat[] sortValueFormats = new DocValueFormat[0]; assert query == searcher.rewrite(query); // already rewritten @@ -229,8 +234,10 @@ public class QueryPhase implements SearchPhase { } assert numDocs > 0; if (searchContext.sort() != null) { - topDocsCollector = TopFieldCollector.create(searchContext.sort(), numDocs, + SortAndFormats sf = searchContext.sort(); + topDocsCollector = TopFieldCollector.create(sf.sort, numDocs, (FieldDoc) after, true, searchContext.trackScores(), searchContext.trackScores()); + sortValueFormats = sf.formats; } else { rescore = !searchContext.rescore().isEmpty(); for (RescoreSearchContext rescoreContext : searchContext.rescore()) { @@ -402,7 +409,7 @@ public class QueryPhase implements SearchPhase { queryResult.terminatedEarly(false); } - queryResult.topDocs(topDocsCallable.call()); + queryResult.topDocs(topDocsCallable.call(), sortValueFormats); if (searchContext.getProfilers() != null) { List shardResults = Profiler.buildShardResults(searchContext.getProfilers().getProfilers()); diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 2b82633ebfd..1408ebe8359 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -19,12 +19,14 @@ package org.elasticsearch.search.query; +import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -51,6 +53,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { private int from; private int size; private TopDocs topDocs; + private DocValueFormat[] sortValueFormats; private InternalAggregations aggregations; private List pipelineAggregators; private Suggest suggest; @@ -112,8 +115,20 @@ public class QuerySearchResult extends QuerySearchResultProvider { return topDocs; } - public void topDocs(TopDocs topDocs) { + public void topDocs(TopDocs topDocs, DocValueFormat[] sortValueFormats) { this.topDocs = topDocs; + if (topDocs.scoreDocs.length > 0 && topDocs.scoreDocs[0] instanceof FieldDoc) { + int numFields = ((FieldDoc) topDocs.scoreDocs[0]).fields.length; + if (numFields != sortValueFormats.length) { + throw new IllegalArgumentException("The number of sort fields does not match: " + + numFields + " != " + sortValueFormats.length); + } + } + this.sortValueFormats = sortValueFormats; + } + + public DocValueFormat[] sortValueFormats() { + return sortValueFormats; } public Aggregations aggregations() { @@ -192,6 +207,15 @@ public class QuerySearchResult extends QuerySearchResultProvider { // shardTarget = readSearchShardTarget(in); from = in.readVInt(); size = in.readVInt(); + int numSortFieldsPlus1 = in.readVInt(); + if (numSortFieldsPlus1 == 0) { + sortValueFormats = null; + } else { + sortValueFormats = new DocValueFormat[numSortFieldsPlus1 - 1]; + for (int i = 0; i < sortValueFormats.length; ++i) { + sortValueFormats[i] = in.readNamedWriteable(DocValueFormat.class); + } + } topDocs = readTopDocs(in); if (in.readBoolean()) { aggregations = InternalAggregations.readAggregations(in); @@ -233,6 +257,14 @@ public class QuerySearchResult extends QuerySearchResultProvider { // shardTarget.writeTo(out); out.writeVInt(from); out.writeVInt(size); + if (sortValueFormats == null) { + out.writeVInt(0); + } else { + out.writeVInt(1 + sortValueFormats.length); + for (int i = 0; i < sortValueFormats.length; ++i) { + out.writeNamedWriteable(sortValueFormats[i]); + } + } writeTopDocs(out, topDocs); if (aggregations == null) { out.writeBoolean(false); diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java index 732a80bad20..b82ed941e1c 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java @@ -61,7 +61,7 @@ public class RescorePhase extends AbstractComponent implements SearchPhase { for (RescoreSearchContext ctx : context.rescore()) { topDocs = ctx.rescorer().rescore(topDocs, context, ctx); } - context.queryResult().topDocs(topDocs); + context.queryResult().topDocs(topDocs, context.queryResult().sortValueFormats()); } catch (IOException e) { throw new ElasticsearchException("Rescore Phase Failed", e); } diff --git a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java index 6d72efdf368..6ed4b0db5bc 100644 --- a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java @@ -20,9 +20,7 @@ package org.elasticsearch.search.searchafter; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; @@ -36,6 +34,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.sort.SortAndFormats; import java.io.IOException; import java.util.ArrayList; @@ -104,21 +104,23 @@ public class SearchAfterBuilder implements ToXContent, Writeable { return Arrays.copyOf(sortValues, sortValues.length); } - public static FieldDoc buildFieldDoc(Sort sort, Object[] values) { - if (sort == null || sort.getSort() == null || sort.getSort().length == 0) { + public static FieldDoc buildFieldDoc(SortAndFormats sort, Object[] values) { + if (sort == null || sort.sort.getSort() == null || sort.sort.getSort().length == 0) { throw new IllegalArgumentException("Sort must contain at least one field."); } - SortField[] sortFields = sort.getSort(); + SortField[] sortFields = sort.sort.getSort(); if (sortFields.length != values.length) { throw new IllegalArgumentException( - SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has " + sort.getSort().length + "."); + SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has " + + sort.sort.getSort().length + "."); } Object[] fieldValues = new Object[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { SortField sortField = sortFields[i]; + DocValueFormat format = sort.formats[i]; if (values[i] != null) { - fieldValues[i] = convertValueFromSortField(values[i], sortField); + fieldValues[i] = convertValueFromSortField(values[i], sortField, format); } else { fieldValues[i] = null; } @@ -130,15 +132,15 @@ public class SearchAfterBuilder implements ToXContent, Writeable { return new FieldDoc(Integer.MAX_VALUE, 0, fieldValues); } - private static Object convertValueFromSortField(Object value, SortField sortField) { + private static Object convertValueFromSortField(Object value, SortField sortField, DocValueFormat format) { if (sortField.getComparatorSource() instanceof IndexFieldData.XFieldComparatorSource) { IndexFieldData.XFieldComparatorSource cmpSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource(); - return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value); + return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value, format); } - return convertValueFromSortType(sortField.getField(), sortField.getType(), value); + return convertValueFromSortType(sortField.getField(), sortField.getType(), value, format); } - private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value) { + private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value, DocValueFormat format) { try { switch (sortType) { case DOC: @@ -179,7 +181,7 @@ public class SearchAfterBuilder implements ToXContent, Writeable { case STRING_VAL: case STRING: - return new BytesRef(value.toString()); + return format.parseBytesRef(value.toString()); default: throw new IllegalArgumentException("Comparator type [" + sortType.name() + "] for field [" + fieldName diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 70876f0acea..892673f890a 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -55,8 +56,10 @@ public class FieldSortBuilder extends SortBuilder { * special field name to sort by index order */ public static final String DOC_FIELD_NAME = "_doc"; - private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC); - private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true); + private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat( + new SortField(null, SortField.Type.DOC), DocValueFormat.RAW); + private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat( + new SortField(null, SortField.Type.DOC, true), DocValueFormat.RAW); private final String fieldName; @@ -246,7 +249,7 @@ public class FieldSortBuilder extends SortBuilder { } @Override - public SortField build(QueryShardContext context) throws IOException { + public SortFieldAndFormat build(QueryShardContext context) throws IOException { if (DOC_FIELD_NAME.equals(fieldName)) { if (order == SortOrder.DESC) { return SORT_DOC_REVERSE; @@ -281,7 +284,8 @@ public class FieldSortBuilder extends SortBuilder { } IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData .comparatorSource(missing, localSortMode, nested); - return new SortField(fieldType.name(), fieldComparatorSource, reverse); + SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse); + return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null)); } } diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 491a37be8b6..dce9a7ec3fe 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -51,6 +51,7 @@ import org.elasticsearch.index.query.GeoValidationMethod; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -504,7 +505,7 @@ public class GeoDistanceSortBuilder extends SortBuilder } @Override - public SortField build(QueryShardContext context) throws IOException { + public SortFieldAndFormat build(QueryShardContext context) throws IOException { final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0); // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes List localPoints = new ArrayList(); @@ -585,7 +586,7 @@ public class GeoDistanceSortBuilder extends SortBuilder }; - return new SortField(fieldName, geoDistanceComparatorSource, reverse); + return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW); } static void parseGeoPoints(XContentParser parser, List geoPoints) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index 8267429f77c..5b9b139e495 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.util.Objects; @@ -40,8 +41,10 @@ public class ScoreSortBuilder extends SortBuilder { public static final String NAME = "_score"; public static final ParseField ORDER_FIELD = new ParseField("order"); - private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); - private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); + private static final SortFieldAndFormat SORT_SCORE = new SortFieldAndFormat( + new SortField(null, SortField.Type.SCORE), DocValueFormat.RAW); + private static final SortFieldAndFormat SORT_SCORE_REVERSE = new SortFieldAndFormat( + new SortField(null, SortField.Type.SCORE, true), DocValueFormat.RAW); /** * Build a ScoreSortBuilder default to descending sort order. @@ -106,7 +109,7 @@ public class ScoreSortBuilder extends SortBuilder { } @Override - public SortField build(QueryShardContext context) { + public SortFieldAndFormat build(QueryShardContext context) { if (order == SortOrder.DESC) { return SORT_SCORE; } else { diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index c574fbcb7df..eeb418c0a97 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -52,6 +52,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -302,7 +303,7 @@ public class ScriptSortBuilder extends SortBuilder { @Override - public SortField build(QueryShardContext context) throws IOException { + public SortFieldAndFormat build(QueryShardContext context) throws IOException { final SearchScript searchScript = context.getScriptService().search( context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), context.getClusterState()); @@ -366,7 +367,7 @@ public class ScriptSortBuilder extends SortBuilder { throw new QueryShardException(context, "custom script sort type [" + type + "] not supported"); } - return new SortField("_script", fieldComparatorSource, reverse); + return new SortFieldAndFormat(new SortField("_script", fieldComparatorSource, reverse), DocValueFormat.RAW); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java b/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java new file mode 100644 index 00000000000..21a9d112fea --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.sort; + +import org.apache.lucene.search.Sort; +import org.elasticsearch.search.DocValueFormat; + +public final class SortAndFormats { + + public final Sort sort; + public final DocValueFormat[] formats; + + public SortAndFormats(Sort sort, DocValueFormat[] formats) { + if (sort.getSort().length != formats.length) { + throw new IllegalArgumentException("Number of sort field mismatch: " + + sort.getSort().length + " != " + formats.length); + } + this.sort = sort; + this.formats = formats; + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index 8cd3fb82413..7fb0baf6548 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.util.ArrayList; @@ -65,9 +66,9 @@ public abstract class SortBuilder> extends ToXContentTo } /** - * Create a @link {@link SortField} from this builder. + * Create a @link {@link SortFieldAndFormat} from this builder. */ - protected abstract SortField build(QueryShardContext context) throws IOException; + protected abstract SortFieldAndFormat build(QueryShardContext context) throws IOException; /** * Set the order of sorting. @@ -143,10 +144,13 @@ public abstract class SortBuilder> extends ToXContentTo } } - public static Optional buildSort(List> sortBuilders, QueryShardContext context) throws IOException { + public static Optional buildSort(List> sortBuilders, QueryShardContext context) throws IOException { List sortFields = new ArrayList<>(sortBuilders.size()); + List sortFormats = new ArrayList<>(sortBuilders.size()); for (SortBuilder builder : sortBuilders) { - sortFields.add(builder.build(context)); + SortFieldAndFormat sf = builder.build(context); + sortFields.add(sf.field); + sortFormats.add(sf.format); } if (!sortFields.isEmpty()) { // optimize if we just sort on score non reversed, we don't really @@ -163,7 +167,9 @@ public abstract class SortBuilder> extends ToXContentTo } } if (sort) { - return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + return Optional.of(new SortAndFormats( + new Sort(sortFields.toArray(new SortField[sortFields.size()])), + sortFormats.toArray(new DocValueFormat[sortFormats.size()]))); } } return Optional.empty(); diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java b/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java new file mode 100644 index 00000000000..f9756b79068 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.sort; + +import java.util.Objects; + +import org.apache.lucene.search.SortField; +import org.elasticsearch.search.DocValueFormat; + +public final class SortFieldAndFormat { + + public final SortField field; + public final DocValueFormat format; + + public SortFieldAndFormat(SortField field, DocValueFormat format) { + this.field = Objects.requireNonNull(field); + this.format = Objects.requireNonNull(format); + } + +} diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java index d62e6ac7dba..4f08eee9b77 100644 --- a/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java +++ b/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java @@ -100,4 +100,5 @@ public class NetworkAddressTests extends ESTestCase { byte bytes[] = InetAddress.getByName(address).getAddress(); return Inet6Address.getByAddress(hostname, bytes, scopeid); } + } diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index 35945f21209..2df15592f45 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -140,8 +140,8 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase DocValueFormat.RAW.parseLong("", randomBoolean(), null)); + expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null)); + + assertEquals(-1d, DocValueFormat.RAW.parseDouble("-1", randomBoolean(), null), 0d); + assertEquals(1d, DocValueFormat.RAW.parseDouble("1", randomBoolean(), null), 0d); + assertEquals(.5, DocValueFormat.RAW.parseDouble("0.5", randomBoolean(), null), 0d); + // not checking exception messages as they could depend on the JVM + expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("", randomBoolean(), null)); + expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null)); + + assertEquals(new BytesRef("abc"), DocValueFormat.RAW.parseBytesRef("abc")); + } + + public void testBooleanParse() { + assertEquals(0L, DocValueFormat.BOOLEAN.parseLong("false", randomBoolean(), null)); + assertEquals(1L, DocValueFormat.BOOLEAN.parseLong("true", randomBoolean(), null)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> DocValueFormat.BOOLEAN.parseLong("", randomBoolean(), null)); + assertEquals("Cannot parse boolean [], expected either [true] or [false]", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, + () -> DocValueFormat.BOOLEAN.parseLong("0", randomBoolean(), null)); + assertEquals("Cannot parse boolean [0], expected either [true] or [false]", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, + () -> DocValueFormat.BOOLEAN.parseLong("False", randomBoolean(), null)); + assertEquals("Cannot parse boolean [False], expected either [true] or [false]", e.getMessage()); + } + + public void testIPParse() { + assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.7"))), + DocValueFormat.IP.parseBytesRef("192.168.1.7")); + assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), + DocValueFormat.IP.parseBytesRef("::1")); + } } diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java index 13c99944959..08c661a130b 100644 --- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java +++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.SearchContextException; import org.elasticsearch.search.SearchHit; @@ -189,11 +188,11 @@ public class SearchAfterIT extends ESIntegTestCase { values.add(randomDouble()); break; case 6: - values.add(new Text(randomAsciiOfLengthBetween(5, 20))); + values.add(randomAsciiOfLengthBetween(5, 20)); break; } } - values.add(new Text(UUIDs.randomBase64UUID())); + values.add(UUIDs.randomBase64UUID()); documents.add(values); } int reqSize = randomInt(NUM_DOCS-1); @@ -296,7 +295,7 @@ public class SearchAfterIT extends ESIntegTestCase { } else if (type == Boolean.class) { mappings.add("field" + Integer.toString(i)); mappings.add("type=boolean"); - } else if (types.get(i) instanceof Text) { + } else if (types.get(i) instanceof String) { mappings.add("field" + Integer.toString(i)); mappings.add("type=keyword"); } else { diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 2978c5dae7a..01a8a07d5bb 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -62,6 +62,7 @@ import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptServiceTests.TestEngineService; import org.elasticsearch.script.ScriptSettings; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -163,12 +164,12 @@ public abstract class AbstractSortTestCase> extends EST QueryShardContext mockShardContext = createMockShardContext(); for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T sortBuilder = createTestItem(); - SortField sortField = sortBuilder.build(mockShardContext); - sortFieldAssertions(sortBuilder, sortField); + SortFieldAndFormat sortField = sortBuilder.build(mockShardContext); + sortFieldAssertions(sortBuilder, sortField.field, sortField.format); } } - protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException; + protected abstract void sortFieldAssertions(T builder, SortField sortField, DocValueFormat format) throws IOException; /** * Test serialization and deserialization of the test sort. diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 4b6eb82304a..baaf3ac5d3c 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.util.Arrays; @@ -110,7 +111,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase` or `bytes[]` have been removed in favor of providing the diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 60a660ead42..5cac904df35 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -18,16 +18,13 @@ */ package org.elasticsearch.test; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -65,13 +62,10 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.threadpool.ThreadPool; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public class TestSearchContext extends SearchContext { final PageCacheRecycler pageCacheRecycler; @@ -365,12 +359,12 @@ public class TestSearchContext extends SearchContext { } @Override - public SearchContext sort(Sort sort) { + public SearchContext sort(SortAndFormats sort) { return null; } @Override - public Sort sort() { + public SortAndFormats sort() { return null; }