From 450db7fcf7b99066b42507be15727856765729ff Mon Sep 17 00:00:00 2001 From: Ekal Golas Date: Mon, 19 Nov 2018 06:21:01 -0800 Subject: [PATCH] [Tests] Fix slowness of AutoDateHistogramAggregatorTests (#35072) Randomize test assertion and test set size instead of asserting on an exhaustive list of dates with fixed test set size. Also refactor common objects used to avoid recreating them, avoid date to string conversion and reduce duplicate test code Closes #33181 --- .../bucket/histogram/package-info.java | 23 + .../AutoDateHistogramAggregatorTests.java | 1665 ++++++----------- 2 files changed, 569 insertions(+), 1119 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/package-info.java diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/package-info.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/package-info.java new file mode 100644 index 00000000000..1e3093bd368 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Histogram module for different bucket specifications used in aggregation. + */ +package org.elasticsearch.search.aggregations.bucket.histogram; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index 1194e6c69d8..2d5109405dc 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -42,1236 +42,668 @@ import org.elasticsearch.search.aggregations.metrics.Stats; import org.hamcrest.Matchers; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.chrono.ISOChronology; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; +import org.junit.Assert; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.Consumer; - -import static org.hamcrest.Matchers.containsString; +import java.util.stream.Collectors; public class AutoDateHistogramAggregatorTests extends AggregatorTestCase { - private static final String DATE_FIELD = "date"; private static final String INSTANT_FIELD = "instant"; - private static final List dataset = Arrays.asList( - "2010-03-12T01:07:45", - "2010-04-27T03:43:34", - "2012-05-18T04:11:00", - "2013-05-29T05:11:31", - "2013-10-31T08:24:05", - "2015-02-13T13:09:32", - "2015-06-24T13:47:43", - "2015-11-13T16:14:34", - "2016-03-04T17:09:50", - "2017-12-12T22:55:46"); + private static final List DATES_WITH_TIME = Arrays.asList( + new DateTime(2010, 3, 12, 1, 7, 45, DateTimeZone.UTC), + new DateTime(2010, 4, 27, 3, 43, 34, DateTimeZone.UTC), + new DateTime(2012, 5, 18, 4, 11, 0, DateTimeZone.UTC), + new DateTime(2013, 5, 29, 5, 11, 31, DateTimeZone.UTC), + new DateTime(2013, 10, 31, 8, 24, 5, DateTimeZone.UTC), + new DateTime(2015, 2, 13, 13, 9, 32, DateTimeZone.UTC), + new DateTime(2015, 6, 24, 13, 47, 43, DateTimeZone.UTC), + new DateTime(2015, 11, 13, 16, 14, 34, DateTimeZone.UTC), + new DateTime(2016, 3, 4, 17, 9, 50, DateTimeZone.UTC), + new DateTime(2017, 12, 12, 22, 55, 46, DateTimeZone.UTC)); + + private static final Query DEFAULT_QUERY = new MatchAllDocsQuery(); public void testMatchNoDocs() throws IOException { - testBothCases(new MatchNoDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), - histogram -> assertEquals(0, histogram.getBuckets().size()) + testBothCases(new MatchNoDocsQuery(), DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), + histogram -> assertEquals(0, histogram.getBuckets().size()) ); } public void testMatchAllDocs() throws IOException { - Query query = new MatchAllDocsQuery(); - - testSearchCase(query, dataset, - aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), - histogram -> assertEquals(10, histogram.getBuckets().size()) + testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), + histogram -> assertEquals(10, histogram.getBuckets().size()) ); - testSearchAndReduceCase(query, dataset, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - histogram -> assertEquals(8, histogram.getBuckets().size()) + testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), + histogram -> assertEquals(8, histogram.getBuckets().size()) ); } public void testSubAggregations() throws IOException { - Query query = new MatchAllDocsQuery(); - testSearchAndReduceCase(query, dataset, - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD) - .subAggregation(AggregationBuilders.stats("stats").field(DATE_FIELD)), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(8, buckets.size()); + testSearchAndReduceCase(DEFAULT_QUERY, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD) + .subAggregation(AggregationBuilders.stats("stats").field(DATE_FIELD)), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(8, buckets.size()); - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2010-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - Stats stats = bucket.getAggregations().get("stats"); - assertEquals("2010-03-12T01:07:45.000Z", stats.getMinAsString()); - assertEquals("2010-04-27T03:43:34.000Z", stats.getMaxAsString()); - assertEquals(2L, stats.getCount()); + Histogram.Bucket bucket = buckets.get(0); + assertEquals("2010-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(2, bucket.getDocCount()); + Stats stats = bucket.getAggregations().get("stats"); + assertEquals("2010-03-12T01:07:45.000Z", stats.getMinAsString()); + assertEquals("2010-04-27T03:43:34.000Z", stats.getMaxAsString()); + assertEquals(2L, stats.getCount()); - bucket = buckets.get(1); - assertEquals("2011-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertTrue(Double.isInfinite(stats.getMin())); - assertTrue(Double.isInfinite(stats.getMax())); - assertEquals(0L, stats.getCount()); + bucket = buckets.get(1); + assertEquals("2011-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(0, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertTrue(Double.isInfinite(stats.getMin())); + assertTrue(Double.isInfinite(stats.getMax())); + assertEquals(0L, stats.getCount()); - bucket = buckets.get(2); - assertEquals("2012-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertEquals("2012-05-18T04:11:00.000Z", stats.getMinAsString()); - assertEquals("2012-05-18T04:11:00.000Z", stats.getMaxAsString()); - assertEquals(1L, stats.getCount()); + bucket = buckets.get(2); + assertEquals("2012-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(1, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertEquals("2012-05-18T04:11:00.000Z", stats.getMinAsString()); + assertEquals("2012-05-18T04:11:00.000Z", stats.getMaxAsString()); + assertEquals(1L, stats.getCount()); - bucket = buckets.get(3); - assertEquals("2013-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertEquals("2013-05-29T05:11:31.000Z", stats.getMinAsString()); - assertEquals("2013-10-31T08:24:05.000Z", stats.getMaxAsString()); - assertEquals(2L, stats.getCount()); + bucket = buckets.get(3); + assertEquals("2013-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(2, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertEquals("2013-05-29T05:11:31.000Z", stats.getMinAsString()); + assertEquals("2013-10-31T08:24:05.000Z", stats.getMaxAsString()); + assertEquals(2L, stats.getCount()); - bucket = buckets.get(4); - assertEquals("2014-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertTrue(Double.isInfinite(stats.getMin())); - assertTrue(Double.isInfinite(stats.getMax())); - assertEquals(0L, stats.getCount()); + bucket = buckets.get(4); + assertEquals("2014-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(0, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertTrue(Double.isInfinite(stats.getMin())); + assertTrue(Double.isInfinite(stats.getMax())); + assertEquals(0L, stats.getCount()); - bucket = buckets.get(5); - assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertEquals("2015-02-13T13:09:32.000Z", stats.getMinAsString()); - assertEquals("2015-11-13T16:14:34.000Z", stats.getMaxAsString()); - assertEquals(3L, stats.getCount()); + bucket = buckets.get(5); + assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(3, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertEquals("2015-02-13T13:09:32.000Z", stats.getMinAsString()); + assertEquals("2015-11-13T16:14:34.000Z", stats.getMaxAsString()); + assertEquals(3L, stats.getCount()); - bucket = buckets.get(6); - assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertEquals("2016-03-04T17:09:50.000Z", stats.getMinAsString()); - assertEquals("2016-03-04T17:09:50.000Z", stats.getMaxAsString()); - assertEquals(1L, stats.getCount()); + bucket = buckets.get(6); + assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(1, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertEquals("2016-03-04T17:09:50.000Z", stats.getMinAsString()); + assertEquals("2016-03-04T17:09:50.000Z", stats.getMaxAsString()); + assertEquals(1L, stats.getCount()); - bucket = buckets.get(7); - assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - stats = bucket.getAggregations().get("stats"); - assertEquals("2017-12-12T22:55:46.000Z", stats.getMinAsString()); - assertEquals("2017-12-12T22:55:46.000Z", stats.getMaxAsString()); - assertEquals(1L, stats.getCount()); - }); + bucket = buckets.get(7); + assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString()); + assertEquals(1, bucket.getDocCount()); + stats = bucket.getAggregations().get("stats"); + assertEquals("2017-12-12T22:55:46.000Z", stats.getMinAsString()); + assertEquals("2017-12-12T22:55:46.000Z", stats.getMaxAsString()); + assertEquals(1L, stats.getCount()); + }); } public void testNoDocs() throws IOException { - Query query = new MatchNoDocsQuery(); - List dates = Collections.emptyList(); - Consumer aggregation = agg -> agg.setNumBuckets(10).field(DATE_FIELD); + final List dates = Collections.emptyList(); + final Consumer aggregation = agg -> agg.setNumBuckets(10).field(DATE_FIELD); - testSearchCase(query, dates, aggregation, - histogram -> assertEquals(0, histogram.getBuckets().size()) + testSearchCase(DEFAULT_QUERY, dates, aggregation, + histogram -> assertEquals(0, histogram.getBuckets().size()) ); - testSearchAndReduceCase(query, dates, aggregation, - histogram -> assertNull(histogram) + testSearchAndReduceCase(DEFAULT_QUERY, dates, aggregation, + Assert::assertNull ); } public void testAggregateWrongField() throws IOException { - testBothCases(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(10).field("wrong_field"), - histogram -> assertEquals(0, histogram.getBuckets().size()) + testBothCases(DEFAULT_QUERY, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(10).field("wrong_field"), + histogram -> assertEquals(0, histogram.getBuckets().size()) ); } public void testIntervalYear() throws IOException { - testSearchCase(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset, - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(5, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2015-02-13T13:09:32.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2015-06-24T13:47:43.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2015-11-13T16:14:34.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2016-03-04T17:09:50.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-12-12T22:55:46.000Z", bucket.getKeyAsString()); + final long start = new DateTime(DateTimeZone.UTC).withDate(2015, 1, 1).getMillis(); + final long end = new DateTime(DateTimeZone.UTC).withDate(2017, 12, 31).getMillis(); + final Query rangeQuery = LongPoint.newRangeQuery(INSTANT_FIELD, start, end); + testSearchCase(rangeQuery, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(5, buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + final Histogram.Bucket bucket = buckets.get(i); + assertEquals(DATES_WITH_TIME.get(5 + i), bucket.getKey()); assertEquals(1, bucket.getDocCount()); } + } ); - testSearchAndReduceCase(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset, - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(3, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - } + testSearchAndReduceCase(rangeQuery, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), + histogram -> { + final DateTime startDate = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(startDate, 3); + expectedDocCount.put(startDate.plusYears(1), 1); + expectedDocCount.put(startDate.plusYears(2), 1); + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } public void testIntervalMonth() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(6, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString()); + final List datesForMonthInterval = Arrays.asList( + new DateTime(2017, 1, 1, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 2, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 3, 4, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 3, 5, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 3, 6, 0, 0, 0, DateTimeZone.UTC)); + testSearchCase(DEFAULT_QUERY, datesForMonthInterval, + aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(datesForMonthInterval.size(), buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + final Histogram.Bucket bucket = buckets.get(i); + assertEquals(datesForMonthInterval.get(i), bucket.getKey()); assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-03-04T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-03-05T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-03-06T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"), - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(3, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-03-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); } + }); + testSearchAndReduceCase(DEFAULT_QUERY, datesForMonthInterval, + aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), + histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(datesForMonthInterval.get(0).withDayOfMonth(1), 1); + expectedDocCount.put(datesForMonthInterval.get(1).withDayOfMonth(1), 2); + expectedDocCount.put(datesForMonthInterval.get(3).withDayOfMonth(1), 3); + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } public void testWithLargeNumberOfBuckets() { - Query query = new MatchAllDocsQuery(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> testSearchCase(query, dataset, - aggregation -> aggregation.setNumBuckets(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS+1).field(DATE_FIELD), + final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> testSearchCase(DEFAULT_QUERY, DATES_WITH_TIME, + aggregation -> aggregation.setNumBuckets(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS + 1).field(DATE_FIELD), // since an exception is thrown, this assertion won't be invoked. - histogram -> assertTrue(false) + histogram -> fail() )); - assertThat(exception.getMessage(), containsString("must be less than")); + assertThat(exception.getMessage(), Matchers.containsString("must be less than")); } public void testIntervalDay() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"), - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(4, buckets.size()); + final List datesForDayInterval = Arrays.asList( + new DateTime(2017, 2, 1, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 2, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 2, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 5, 0, 0, 0, DateTimeZone.UTC)); + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(datesForDayInterval.get(0), 1); + expectedDocCount.put(datesForDayInterval.get(1), 2); + expectedDocCount.put(datesForDayInterval.get(3), 3); + expectedDocCount.put(datesForDayInterval.get(6), 1); - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01", - "2017-02-02", - "2017-02-02", - "2017-02-03", - "2017-02-03", - "2017-02-03", - "2017-02-05" - ), - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(5, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-04T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - } + testSearchCase(DEFAULT_QUERY, datesForDayInterval, + aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + }); + testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(5, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } public void testIntervalDayWithTZ() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"), - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(4, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-01-31T23:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T23:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-02T23:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-04T23:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"), - aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(5, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-01-31T00:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T00:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-02T00:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-03T00:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-04T00:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - }); + final List datesForDayInterval = Arrays.asList( + new DateTime(2017, 2, 1, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 2, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 2, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 3, 0, 0, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 5, 0, 0, 0, DateTimeZone.UTC)); + testSearchCase(DEFAULT_QUERY, datesForDayInterval, + aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put("2017-01-31T23:00:00.000-01:00", 1); + expectedDocCount.put("2017-02-01T23:00:00.000-01:00", 2); + expectedDocCount.put("2017-02-02T23:00:00.000-01:00", 3); + expectedDocCount.put("2017-02-04T23:00:00.000-01:00", 1); + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKeyAsString(), 0).longValue(), bucket.getDocCount())); + }); + testSearchAndReduceCase(DEFAULT_QUERY, datesForDayInterval, + aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put("2017-01-31T00:00:00.000-01:00", 1); + expectedDocCount.put("2017-02-01T00:00:00.000-01:00", 2); + expectedDocCount.put("2017-02-02T00:00:00.000-01:00", 3); + expectedDocCount.put("2017-02-04T00:00:00.000-01:00", 1); + final List buckets = histogram.getBuckets(); + assertEquals(5, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKeyAsString(), 0).longValue(), bucket.getDocCount())); + }); } public void testIntervalHour() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(10, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T09:35:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T10:15:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T13:06:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T14:04:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T14:05:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T15:59:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T16:06:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(8); - assertEquals("2017-02-01T16:48:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(9); - assertEquals("2017-02-01T16:59:00.000Z", bucket.getKeyAsString()); + final List datesForHourInterval = Arrays.asList( + new DateTime(2017, 2, 1, 9, 2, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 35, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 10, 15, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 13, 6, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 14, 4, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 14, 5, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 15, 59, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 6, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 48, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 59, 0, DateTimeZone.UTC)); + testSearchCase(DEFAULT_QUERY, datesForHourInterval, + aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(datesForHourInterval.size(), buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + final Histogram.Bucket bucket = buckets.get(i); + assertEquals(datesForHourInterval.get(i), bucket.getKey()); assertEquals(1, bucket.getDocCount()); } + } ); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), - aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(8, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T11:00:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T12:00:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - } + testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), + histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(datesForHourInterval.get(0).withMinuteOfHour(0), 2); + expectedDocCount.put(datesForHourInterval.get(2).withMinuteOfHour(0), 1); + expectedDocCount.put(datesForHourInterval.get(3).withMinuteOfHour(0), 1); + expectedDocCount.put(datesForHourInterval.get(4).withMinuteOfHour(0), 2); + expectedDocCount.put(datesForHourInterval.get(6).withMinuteOfHour(0), 1); + expectedDocCount.put(datesForHourInterval.get(7).withMinuteOfHour(0), 3); + final List buckets = histogram.getBuckets(); + assertEquals(8, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } + ); + testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), + histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(datesForHourInterval.get(0).withMinuteOfHour(0), 3); + expectedDocCount.put(datesForHourInterval.get(0).plusHours(3).withMinuteOfHour(0), 3); + expectedDocCount.put(datesForHourInterval.get(0).plusHours(6).withMinuteOfHour(0), 4); + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } public void testIntervalHourWithTZ() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(10, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T08:02:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T08:35:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T09:15:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T12:06:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T13:04:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T13:05:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T14:59:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T15:06:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(8); - assertEquals("2017-02-01T15:48:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(9); - assertEquals("2017-02-01T15:59:00.000-01:00", bucket.getKeyAsString()); + final List datesForHourInterval = Arrays.asList( + new DateTime(2017, 2, 1, 9, 2, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 35, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 10, 15, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 13, 6, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 14, 4, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 14, 5, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 15, 59, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 6, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 48, 0, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 16, 59, 0, DateTimeZone.UTC)); + testSearchCase(DEFAULT_QUERY, datesForHourInterval, + aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), + histogram -> { + final List dateStrings = datesForHourInterval.stream() + .map(dateTime -> dateTime.withZone(DateTimeZone.forOffsetHours(-1)).toString()).collect(Collectors.toList()); + final List buckets = histogram.getBuckets(); + assertEquals(datesForHourInterval.size(), buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + final Histogram.Bucket bucket = buckets.get(i); + assertEquals(dateStrings.get(i), bucket.getKeyAsString()); assertEquals(1, bucket.getDocCount()); } + } ); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), - aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(8, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T08:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T09:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T10:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T11:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T12:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T13:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T14:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T15:00:00.000-01:00", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - } + testSearchAndReduceCase(DEFAULT_QUERY, datesForHourInterval, + aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), + histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put("2017-02-01T08:00:00.000-01:00", 2); + expectedDocCount.put("2017-02-01T09:00:00.000-01:00", 1); + expectedDocCount.put("2017-02-01T12:00:00.000-01:00", 1); + expectedDocCount.put("2017-02-01T13:00:00.000-01:00", 2); + expectedDocCount.put("2017-02-01T14:00:00.000-01:00", 1); + expectedDocCount.put("2017-02-01T15:00:00.000-01:00", 3); + final List buckets = histogram.getBuckets(); + assertEquals(8, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKeyAsString(), 0).longValue(), bucket.getDocCount())); + } ); } - public void testAllSecondIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 600; i++) { - DateTime date = startDate.plusSeconds(i); - dataset.add(format.print(date)); + public void testRandomSecondIntervals() throws IOException { + final int length = 120; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusSeconds(i); + dataset.add(date); } - - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(600, buckets.size()); - for (int i = 0; i < 600; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusSeconds(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } - }); - - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(120, buckets.size()); - for (int i = 0; i < 120; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusSeconds(i * 5), bucket.getKey()); - assertEquals(5, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(60, buckets.size()); - for (int i = 0; i < 60; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusSeconds(i * 10), bucket.getKey()); - assertEquals(10, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(20, buckets.size()); - for (int i = 0; i < 20; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusSeconds(i * 30), bucket.getKey()); - assertEquals(30, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(10, buckets.size()); - for (int i = 0; i < 10; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMinutes(i), bucket.getKey()); - assertEquals(60, bucket.getDocCount()); - } - }); + final Map bucketsToExpectedDocCountMap = new HashMap<>(); + bucketsToExpectedDocCountMap.put(120, 1); + bucketsToExpectedDocCountMap.put(60, 5); + bucketsToExpectedDocCountMap.put(20, 10); + bucketsToExpectedDocCountMap.put(10, 30); + bucketsToExpectedDocCountMap.put(3, 60); + final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + final int expectedDocCount = randomEntry.getValue(); + final int expectedSize = length / expectedDocCount; + assertEquals(expectedSize, buckets.size()); + final int randomIndex = randomInt(expectedSize - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusSeconds(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); + }); } - public void testAllMinuteIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 600; i++) { - DateTime date = startDate.plusMinutes(i); - dataset.add(format.print(date)); + public void testRandomMinuteIntervals() throws IOException { + final int length = 120; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusMinutes(i); + dataset.add(date); } - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(600, buckets.size()); - for (int i = 0; i < 600; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMinutes(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(120, buckets.size()); - for (int i = 0; i < 120; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMinutes(i * 5), bucket.getKey()); - assertEquals(5, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(60, buckets.size()); - for (int i = 0; i < 60; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMinutes(i * 10), bucket.getKey()); - assertEquals(10, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(20, buckets.size()); - for (int i = 0; i < 20; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMinutes(i * 30), bucket.getKey()); - assertEquals(30, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(10, buckets.size()); - for (int i = 0; i < 10; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusHours(i), bucket.getKey()); - assertEquals(60, bucket.getDocCount()); - } - }); + final Map bucketsToExpectedDocCountMap = new HashMap<>(); + bucketsToExpectedDocCountMap.put(120, 1); + bucketsToExpectedDocCountMap.put(60, 5); + bucketsToExpectedDocCountMap.put(20, 10); + bucketsToExpectedDocCountMap.put(10, 30); + bucketsToExpectedDocCountMap.put(3, 60); + final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + final int expectedDocCount = randomEntry.getValue(); + final int expectedSize = length / expectedDocCount; + assertEquals(expectedSize, buckets.size()); + final int randomIndex = randomInt(expectedSize - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusMinutes(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); + }); } - public void testAllHourIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 600; i++) { - DateTime date = startDate.plusHours(i); - dataset.add(format.print(date)); + public void testRandomHourIntervals() throws IOException { + final int length = 72; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusHours(i); + dataset.add(date); } - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(600, buckets.size()); - for (int i = 0; i < 600; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusHours(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(200, buckets.size()); - for (int i = 0; i < 200; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusHours(i * 3), bucket.getKey()); - assertEquals(3, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(50, buckets.size()); - for (int i = 0; i < 50; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusHours(i * 12), bucket.getKey()); - assertEquals(12, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(30).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(25, buckets.size()); - for (int i = 0; i < 25; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusDays(i), bucket.getKey()); - assertEquals(24, bucket.getDocCount()); - } - }); + final Map bucketsToExpectedDocCountMap = new HashMap<>(); + bucketsToExpectedDocCountMap.put(72, 1); + bucketsToExpectedDocCountMap.put(36, 3); + bucketsToExpectedDocCountMap.put(12, 12); + bucketsToExpectedDocCountMap.put(3, 24); + final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + final int expectedDocCount = randomEntry.getValue(); + final int expectedSize = length / expectedDocCount; + assertEquals(expectedSize, buckets.size()); + final int randomIndex = randomInt(expectedSize - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusHours(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); + }); } - public void testAllDayIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 700; i++) { - DateTime date = startDate.plusDays(i); - dataset.add(format.print(date)); + public void testRandomDayIntervals() throws IOException { + final int length = 140; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusDays(i); + dataset.add(date); } - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(700).field(DATE_FIELD), + final int randomChoice = randomIntBetween(1, 3); + if (randomChoice == 1) { + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(length).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(700, buckets.size()); - for (int i = 0; i < 700; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusDays(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } + final List buckets = histogram.getBuckets(); + assertEquals(length, buckets.size()); + final int randomIndex = randomInt(length - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusDays(randomIndex), bucket.getKey()); + assertEquals(1, bucket.getDocCount()); }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(100, buckets.size()); - for (int i = 0; i < 100; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusDays(i * 7), bucket.getKey()); - assertEquals(7, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(30).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(24, buckets.size()); - for (int i = 0; i < 24; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMonths(i), bucket.getKey()); - assertThat(bucket.getDocCount(), Matchers.lessThanOrEqualTo(31L)); - } - }); - } - - public void testAllMonthIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 600; i++) { - DateTime date = startDate.plusMonths(i); - dataset.add(format.print(date)); - } - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(600, buckets.size()); - for (int i = 0; i < 600; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMonths(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, - aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(200, buckets.size()); - for (int i = 0; i < 200; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusMonths(i * 3), bucket.getKey()); - assertEquals(3, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, + } else if (randomChoice == 2) { + testSearchAndReduceCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(60).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(50, buckets.size()); - for (int i = 0; i < 50; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i), bucket.getKey()); - assertEquals(12, bucket.getDocCount()); - } + final List buckets = histogram.getBuckets(); + final int expectedDocCount = 7; + assertEquals(20, buckets.size()); + final int randomIndex = randomInt(19); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusDays(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); }); - } - - public void testAllYearIntervals() throws IOException { - DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - List dataset = new ArrayList<>(); - DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC()); - for (int i = 0; i < 600; i++) { - DateTime date = startDate.plusYears(i); - dataset.add(format.print(date)); - } - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(600, buckets.size()); - for (int i = 0; i < 600; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i), bucket.getKey()); - assertEquals(1, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(120, buckets.size()); - for (int i = 0; i < 120; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i * 5), bucket.getKey()); - assertEquals(5, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(60, buckets.size()); - for (int i = 0; i < 60; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i * 10), bucket.getKey()); - assertEquals(10, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(30, buckets.size()); - for (int i = 0; i < 30; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i * 20), bucket.getKey()); - assertEquals(20, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(20).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(12, buckets.size()); - for (int i = 0; i < 12; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i * 50), bucket.getKey()); - assertEquals(50, bucket.getDocCount()); - } - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(6, buckets.size()); - for (int i = 0; i < 6; i++) { - Histogram.Bucket bucket = buckets.get(i); - assertEquals(startDate.plusYears(i * 100), bucket.getKey()); - assertEquals(100, bucket.getDocCount()); - } - }); - } - - public void testInterval3Hour() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), - aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(10, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T09:35:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T10:15:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T13:06:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T14:04:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T14:05:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T15:59:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T16:06:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(8); - assertEquals("2017-02-01T16:48:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(9); - assertEquals("2017-02-01T16:59:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - } - ); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:00.000Z", - "2017-02-01T09:35:00.000Z", - "2017-02-01T10:15:00.000Z", - "2017-02-01T13:06:00.000Z", - "2017-02-01T14:04:00.000Z", - "2017-02-01T14:05:00.000Z", - "2017-02-01T15:59:00.000Z", - "2017-02-01T16:06:00.000Z", - "2017-02-01T16:48:00.000Z", - "2017-02-01T16:59:00.000Z" - ), + } else if (randomChoice == 3) { + testSearchAndReduceCase(DEFAULT_QUERY, dataset, aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(3, buckets.size()); + final List buckets = histogram.getBuckets(); + assertEquals(5, buckets.size()); + final int randomIndex = randomInt(2); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusMonths(randomIndex), bucket.getKey()); + assertEquals(startDate.plusMonths(randomIndex).dayOfMonth().getMaximumValue(), bucket.getDocCount()); + }); + } + } - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); + public void testRandomMonthIntervals() throws IOException { + final int length = 60; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusMonths(i); + dataset.add(date); + } + final Map bucketsToExpectedDocCountMap = new HashMap<>(); + bucketsToExpectedDocCountMap.put(60, 1); + bucketsToExpectedDocCountMap.put(30, 3); + bucketsToExpectedDocCountMap.put(6, 12); + final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + final int expectedDocCount = randomEntry.getValue(); + final int expectedSize = length / expectedDocCount; + assertEquals(expectedSize, buckets.size()); + final int randomIndex = randomInt(expectedSize - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusMonths(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); + }); + } - bucket = buckets.get(1); - assertEquals("2017-02-01T12:00:00.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString()); - assertEquals(4, bucket.getDocCount()); - } - ); + public void testRandomYearIntervals() throws IOException { + final int length = 300; + final List dataset = new ArrayList<>(length); + final DateTime startDate = new DateTime(2017, 1, 1, 0, 0, DateTimeZone.UTC); + for (int i = 0; i < length; i++) { + final DateTime date = startDate.plusYears(i); + dataset.add(date); + } + final Map bucketsToExpectedDocCountMap = new HashMap<>(); + bucketsToExpectedDocCountMap.put(300, 1); + bucketsToExpectedDocCountMap.put(150, 5); + bucketsToExpectedDocCountMap.put(50, 10); + bucketsToExpectedDocCountMap.put(25, 20); + bucketsToExpectedDocCountMap.put(10, 50); + bucketsToExpectedDocCountMap.put(5, 100); + final Map.Entry randomEntry = randomFrom(bucketsToExpectedDocCountMap.entrySet()); + testSearchAndReduceCase(DEFAULT_QUERY, dataset, + aggregation -> aggregation.setNumBuckets(randomEntry.getKey()).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + final int expectedDocCount = randomEntry.getValue(); + final int expectedSize = length / expectedDocCount; + assertEquals(expectedSize, buckets.size()); + final int randomIndex = randomInt(expectedSize - 1); + final Histogram.Bucket bucket = buckets.get(randomIndex); + assertEquals(startDate.plusYears(randomIndex * expectedDocCount), bucket.getKey()); + assertEquals(expectedDocCount, bucket.getDocCount()); + }); } public void testIntervalMinute() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:35.000Z", - "2017-02-01T09:02:59.000Z", - "2017-02-01T09:15:37.000Z", - "2017-02-01T09:16:04.000Z", - "2017-02-01T09:16:42.000Z" - ), - aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(5, buckets.size()); + final List datesForMinuteInterval = Arrays.asList( + new DateTime(2017, 2, 1, 9, 2, 35, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 2, 59, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 15, 37, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 16, 4, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 9, 16, 42, DateTimeZone.UTC)); - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:02:35.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T09:02:59.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T09:15:37.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T09:16:04.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T09:16:42.000Z", bucket.getKeyAsString()); + testSearchCase(DEFAULT_QUERY, datesForMinuteInterval, + aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(datesForMinuteInterval.size(), buckets.size()); + for (int i = 0; i < buckets.size(); i++) { + final Histogram.Bucket bucket = buckets.get(i); + assertEquals(datesForMinuteInterval.get(i), bucket.getKey()); assertEquals(1, bucket.getDocCount()); } + } ); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T09:02:35.000Z", - "2017-02-01T09:02:59.000Z", - "2017-02-01T09:15:37.000Z", - "2017-02-01T09:16:04.000Z", - "2017-02-01T09:16:42.000Z" - ), - aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(15, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T09:03:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T09:04:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T09:05:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T09:06:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T09:07:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T09:08:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(7); - assertEquals("2017-02-01T09:09:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(8); - assertEquals("2017-02-01T09:10:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(9); - assertEquals("2017-02-01T09:11:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(10); - assertEquals("2017-02-01T09:12:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(11); - assertEquals("2017-02-01T09:13:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(12); - assertEquals("2017-02-01T09:14:00.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(13); - assertEquals("2017-02-01T09:15:00.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(14); - assertEquals("2017-02-01T09:16:00.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - } + testSearchAndReduceCase(DEFAULT_QUERY, datesForMinuteInterval, + aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD), + histogram -> { + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(datesForMinuteInterval.get(0).withSecondOfMinute(0), 2); + expectedDocCount.put(datesForMinuteInterval.get(2).withSecondOfMinute(0), 1); + expectedDocCount.put(datesForMinuteInterval.get(3).withSecondOfMinute(0), 2); + final List buckets = histogram.getBuckets(); + assertEquals(15, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } public void testIntervalSecond() throws IOException { - testSearchCase(new MatchAllDocsQuery(), - Arrays.asList("2017-02-01T00:00:05.015Z", "2017-02-01T00:00:07.299Z", "2017-02-01T00:00:07.074Z", - "2017-02-01T00:00:11.688Z", "2017-02-01T00:00:11.210Z", "2017-02-01T00:00:11.380Z"), - aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(3, buckets.size()); + final List datesForSecondInterval = Arrays.asList( + new DateTime(2017, 2, 1, 0, 0, 5, 15, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 0, 0, 7, 299, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 0, 0, 7, 74, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 0, 0, 11, 688, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 0, 0, 11, 210, DateTimeZone.UTC), + new DateTime(2017, 2, 1, 0, 0, 11, 380, DateTimeZone.UTC)); + final DateTime startDate = datesForSecondInterval.get(0).withMillisOfSecond(0); + final Map expectedDocCount = new HashMap<>(); + expectedDocCount.put(startDate, 1); + expectedDocCount.put(startDate.plusSeconds(2), 2); + expectedDocCount.put(startDate.plusSeconds(6), 3); - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T00:00:07.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - }); - testSearchAndReduceCase(new MatchAllDocsQuery(), - Arrays.asList( - "2017-02-01T00:00:05.015Z", - "2017-02-01T00:00:07.299Z", - "2017-02-01T00:00:07.074Z", - "2017-02-01T00:00:11.688Z", - "2017-02-01T00:00:11.210Z", - "2017-02-01T00:00:11.380Z" - ), - aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), - histogram -> { - List buckets = histogram.getBuckets(); - assertEquals(7, buckets.size()); - - Histogram.Bucket bucket = buckets.get(0); - assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString()); - assertEquals(1, bucket.getDocCount()); - - bucket = buckets.get(1); - assertEquals("2017-02-01T00:00:06.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(2); - assertEquals("2017-02-01T00:00:07.000Z", bucket.getKeyAsString()); - assertEquals(2, bucket.getDocCount()); - - bucket = buckets.get(3); - assertEquals("2017-02-01T00:00:08.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(4); - assertEquals("2017-02-01T00:00:09.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(5); - assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString()); - assertEquals(0, bucket.getDocCount()); - - bucket = buckets.get(6); - assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString()); - assertEquals(3, bucket.getDocCount()); - } + testSearchCase(DEFAULT_QUERY, datesForSecondInterval, + aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCount.size(), buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + }); + testSearchAndReduceCase(DEFAULT_QUERY, datesForSecondInterval, + aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(7, buckets.size()); + buckets.forEach(bucket -> + assertEquals(expectedDocCount.getOrDefault(bucket.getKey(), 0).longValue(), bucket.getDocCount())); + } ); } - private void testSearchCase(Query query, List dataset, - Consumer configure, - Consumer verify) throws IOException { + private void testSearchCase(final Query query, final List dataset, + final Consumer configure, + final Consumer verify) throws IOException { executeTestCase(false, query, dataset, configure, verify); } - private void testSearchAndReduceCase(Query query, List dataset, - Consumer configure, - Consumer verify) throws IOException { + private void testSearchAndReduceCase(final Query query, final List dataset, + final Consumer configure, + final Consumer verify) throws IOException { executeTestCase(true, query, dataset, configure, verify); } - private void testBothCases(Query query, List dataset, - Consumer configure, - Consumer verify) throws IOException { - testSearchCase(query, dataset, configure, verify); - testSearchAndReduceCase(query, dataset, configure, verify); + private void testBothCases(final Query query, final List dataset, + final Consumer configure, + final Consumer verify) throws IOException { + executeTestCase(false, query, dataset, configure, verify); + executeTestCase(true, query, dataset, configure, verify); } @Override protected IndexSettings createIndexSettings() { - Settings nodeSettings = Settings.builder() - .put("search.max_buckets", 100000).build(); + final Settings nodeSettings = Settings.builder() + .put("search.max_buckets", 25000).build(); return new IndexSettings( IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) .numberOfShards(1) @@ -1282,19 +714,18 @@ public class AutoDateHistogramAggregatorTests extends AggregatorTestCase { ); } - private void executeTestCase(boolean reduced, Query query, List dataset, - Consumer configure, - Consumer verify) throws IOException { - + private void executeTestCase(final boolean reduced, final Query query, final List dataset, + final Consumer configure, + final Consumer verify) throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { - Document document = new Document(); - for (String date : dataset) { + final Document document = new Document(); + for (final DateTime date : dataset) { if (frequently()) { indexWriter.commit(); } - long instant = asLong(date); + final long instant = date.getMillis(); document.add(new SortedNumericDocValuesField(DATE_FIELD, instant)); document.add(new LongPoint(INSTANT_FIELD, instant)); indexWriter.addDocument(document); @@ -1303,19 +734,19 @@ public class AutoDateHistogramAggregatorTests extends AggregatorTestCase { } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + final IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - AutoDateHistogramAggregationBuilder aggregationBuilder = new AutoDateHistogramAggregationBuilder("_name"); + final AutoDateHistogramAggregationBuilder aggregationBuilder = new AutoDateHistogramAggregationBuilder("_name"); if (configure != null) { configure.accept(aggregationBuilder); } - DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name"); - DateFieldMapper.DateFieldType fieldType = builder.fieldType(); + final DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name"); + final DateFieldMapper.DateFieldType fieldType = builder.fieldType(); fieldType.setHasDocValues(true); fieldType.setName(aggregationBuilder.field()); - InternalAutoDateHistogram histogram; + final InternalAutoDateHistogram histogram; if (reduced) { histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); } else { @@ -1325,8 +756,4 @@ public class AutoDateHistogramAggregatorTests extends AggregatorTestCase { } } } - - private static long asLong(String dateTime) { - return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(dateTime).getMillis(); - } }