diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index aa283cda4c2..9f0273fbc02 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -1,4 +1,4 @@ -setup: +"search with search_after parameter": - do: indices.create: index: test @@ -24,9 +24,6 @@ setup: indices.refresh: index: test ---- -"search with search_after parameter": - - do: search: rest_total_hits_as_int: true @@ -97,3 +94,137 @@ setup: - match: {hits.total: 3} - length: {hits.hits: 0 } + +--- +"date": + - skip: + version: " - 7.9.99" + reason: fixed in 7.10.0 + + - do: + indices.create: + index: test + body: + mappings: + properties: + timestamp: + type: date + format: yyyy-MM-dd HH:mm:ss.SSS + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"timestamp":"2019-10-21 00:30:04.828"} + {"index":{}} + {"timestamp":"2019-10-21 08:30:04.828"} + + - do: + search: + index: test + rest_total_hits_as_int: true + body: + size: 1 + sort: [{ timestamp: desc }] + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828" } + - match: {hits.hits.0.sort: [1571646604828] } + + # search_after with the sort + - do: + search: + index: test + rest_total_hits_as_int: true + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: [1571646604828] + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" } + - match: {hits.hits.0.sort: [1571617804828] } + + # search_after with the formatted date + - do: + search: + index: test + rest_total_hits_as_int: true + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: ["2019-10-21 08:30:04.828"] + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" } + - match: {hits.hits.0.sort: [1571617804828] } + +--- +"date_nanos": + - skip: + version: " - 7.9.99" + reason: fixed in 7.10.0 + + - do: + indices.create: + index: test + body: + mappings: + properties: + timestamp: + type: date_nanos + format: yyyy-MM-dd HH:mm:ss.SSSSSS + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"timestamp":"2019-10-21 00:30:04.828740"} + {"index":{}} + {"timestamp":"2019-10-21 08:30:04.828733"} + + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828733" } + - match: {hits.hits.0.sort: [1571646604828733000] } + + # search_after with the sort + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: [1571646604828733000] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } + - match: {hits.hits.0.sort: [1571617804828740000] } + + # search_after with the formatted date + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: ["2019-10-21 08:30:04.828733"] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } + - match: {hits.hits.0.sort: [1571617804828740000] } + diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index a5597d366bb..41441ac36fc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -100,7 +100,7 @@ public class SortedNumericIndexFieldData extends IndexNumericFieldData { @Override protected XFieldComparatorSource dateComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { if (numericType == NumericType.DATE_NANOSECONDS) { - // converts date values to nanosecond resolution + // converts date_nanos values to millisecond resolution return new LongValuesComparatorSource(this, missingValue, sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds)); } @@ -110,7 +110,7 @@ public class SortedNumericIndexFieldData extends IndexNumericFieldData { @Override protected XFieldComparatorSource dateNanosComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { if (numericType == NumericType.DATE) { - // converts date_nanos values to millisecond resolution + // converts date values to nanosecond resolution return new LongValuesComparatorSource(this, missingValue, sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 6ba9e1fb94f..db11c044eed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -451,6 +451,7 @@ public final class DateFieldMapper extends ParametrizedFieldMapper { } // the resolution here is always set to milliseconds, as aggregations use this formatter mainly and those are always in // milliseconds. The only special case here is docvalue fields, which are handled somewhere else + // TODO maybe aggs should force millis because lots so of other places want nanos? return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index a98f89ba4c0..4078d5f664c 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -274,6 +274,11 @@ public interface DocValueFormat extends NamedWriteable { public double parseDouble(String value, boolean roundUp, LongSupplier now) { return parseLong(value, roundUp, now); } + + @Override + public String toString() { + return "DocValueFormat.DateTime(" + formatter + ", " + timeZone + ", " + resolution + ")"; + } } DocValueFormat GEOHASH = new DocValueFormat() { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 47e2813ac08..eae89eea341 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -404,6 +404,7 @@ public class FieldSortBuilder extends SortBuilder { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); } final SortField field; + boolean isNanosecond = false; if (numericType != null) { if (fieldData instanceof IndexNumericFieldData == false) { throw new QueryShardException(context, @@ -414,8 +415,15 @@ public class FieldSortBuilder extends SortBuilder { field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse); } else { field = fieldData.sortField(missing, localSortMode(), nested, reverse); + if (fieldData instanceof IndexNumericFieldData) { + isNanosecond = ((IndexNumericFieldData) fieldData).getNumericType() == NumericType.DATE_NANOSECONDS; + } } - return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null)); + DocValueFormat format = fieldType.docValueFormat(null, null); + if (isNanosecond) { + format = DocValueFormat.withNanosecondResolution(format); + } + return new SortFieldAndFormat(field, format); } public boolean canRewriteToMatchNone() { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index a5e8eb8fa9c..b1bcbcad629 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -80,7 +80,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase