Add randomiziation to test for derivative aggregation

src/test/java/org/elasticsearch/search/aggregations/reducers/DerivativeTests.java

@@ -21,16 +21,20 @@ package org.elasticsearch.search.aggregations.reducers;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.Bucket;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
+import org.elasticsearch.search.aggregations.reducers.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -41,7 +45,6 @@ import static org.elasticsearch.search.aggregations.reducers.ReducerBuilders.der
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 import static org.hamcrest.core.IsNull.nullValue;
 
@@ -49,49 +52,44 @@ import static org.hamcrest.core.IsNull.nullValue;
 public class DerivativeTests extends ElasticsearchIntegrationTest {
 
     private static final String SINGLE_VALUED_FIELD_NAME = "l_value";
-    private static final String MULTI_VALUED_FIELD_NAME = "l_values";
 
-    static int numDocs;
+    private static int interval;
-    static int interval;
+    private static int numValueBuckets;
-    static int numValueBuckets, numValuesBuckets;
+    private static int numFirstDerivValueBuckets;
-    static int numFirstDerivValueBuckets, numFirstDerivValuesBuckets;
+    private static int numSecondDerivValueBuckets;
-    static int numSecondDerivValueBuckets;
+    private static long[] valueCounts;
-    static long[] valueCounts, valuesCounts;
+    private static long[] firstDerivValueCounts;
-    static long[] firstDerivValueCounts, firstDerivValuesCounts;
+    private static long[] secondDerivValueCounts;
-    static long[] secondDerivValueCounts;
+
+    private static Long[] valueCounts_empty;
+    private static long numDocsEmptyIdx;
+    private static Double[] firstDerivValueCounts_empty;
+
+    // expected bucket values for random setup with gaps
+    private static int numBuckets_empty_rnd;
+    private static Long[] valueCounts_empty_rnd;
+    private static Double[] firstDerivValueCounts_empty_rnd;
+    private static long numDocsEmptyIdx_rnd;
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
         createIndex("idx");
         createIndex("idx_unmapped");
 
-        numDocs = randomIntBetween(6, 20);
+        interval = 5;
-        interval = randomIntBetween(2, 5);
+        numValueBuckets = randomIntBetween(6, 80);
 
-        numValueBuckets = numDocs / interval + 1;
         valueCounts = new long[numValueBuckets];
-        for (int i = 0; i < numDocs; i++) {
+        for (int i = 0; i < numValueBuckets; i++) {
-            final int bucket = (i + 1) / interval;
+            valueCounts[i] = randomIntBetween(1, 20);
-            valueCounts[bucket]++;
-        }
-
-        numValuesBuckets = (numDocs + 1) / interval + 1;
-        valuesCounts = new long[numValuesBuckets];
-        for (int i = 0; i < numDocs; i++) {
-            final int bucket1 = (i + 1) / interval;
-            final int bucket2 = (i + 2) / interval;
-            valuesCounts[bucket1]++;
-            if (bucket1 != bucket2) {
-                valuesCounts[bucket2]++;
-            }
         }
 
         numFirstDerivValueBuckets = numValueBuckets - 1;
         firstDerivValueCounts = new long[numFirstDerivValueBuckets];
-        long lastValueCount = -1;
+        Long lastValueCount = null;
         for (int i = 0; i < numValueBuckets; i++) {
             long thisValue = valueCounts[i];
-            if (lastValueCount != -1) {
+            if (lastValueCount != null) {
                 long diff = thisValue - lastValueCount;
                 firstDerivValueCounts[i - 1] = diff;
             }
@@ -100,112 +98,69 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
         numSecondDerivValueBuckets = numFirstDerivValueBuckets - 1;
         secondDerivValueCounts = new long[numSecondDerivValueBuckets];
-        long lastFirstDerivativeValueCount = -1;
+        Long lastFirstDerivativeValueCount = null;
         for (int i = 0; i < numFirstDerivValueBuckets; i++) {
             long thisFirstDerivativeValue = firstDerivValueCounts[i];
-            if (lastFirstDerivativeValueCount != -1) {
+            if (lastFirstDerivativeValueCount != null) {
                 long diff = thisFirstDerivativeValue - lastFirstDerivativeValueCount;
                 secondDerivValueCounts[i - 1] = diff;
             }
             lastFirstDerivativeValueCount = thisFirstDerivativeValue;
         }
 
-        numFirstDerivValuesBuckets = numValuesBuckets - 1;
-        firstDerivValuesCounts = new long[numFirstDerivValuesBuckets];
-        long lastValuesCount = -1;
-        for (int i = 0; i < numValuesBuckets; i++) {
-            long thisValue = valuesCounts[i];
-            if (lastValuesCount != -1) {
-                long diff = thisValue - lastValuesCount;
-                firstDerivValuesCounts[i - 1] = diff;
-            }
-            lastValuesCount = thisValue;
-        }
-
         List<IndexRequestBuilder> builders = new ArrayList<>();
-
+        for (int i = 0; i < numValueBuckets; i++) {
-        for (int i = 0; i < numDocs; i++) {
+            for (int docs = 0; docs < valueCounts[i]; docs++) {
-            builders.add(client().prepareIndex("idx", "type").setSource(
+                builders.add(client().prepareIndex("idx", "type").setSource(newDocBuilder(i * interval)));
-                    jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, i + 1).startArray(MULTI_VALUED_FIELD_NAME).value(i + 1)
+            }
-                            .value(i + 2).endArray().field("tag", "tag" + i).endObject()));
         }
 
+        // setup for index with empty buckets
+        valueCounts_empty = new Long[] { 1l, 1l, 2l, 0l, 2l, 2l, 0l, 0l, 0l, 3l, 2l, 1l };
+        firstDerivValueCounts_empty = new Double[] { null, 0d, 1d, -2d, 2d, 0d, -2d, 0d, 0d, 3d, -1d, -1d };
+
         assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=integer"));
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 0).setSource(
+        for (int i = 0; i < valueCounts_empty.length; i++) {
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 0).endObject()));
+            for (int docs = 0; docs < valueCounts_empty[i]; docs++) {
+                builders.add(client().prepareIndex("empty_bucket_idx", "type").setSource(newDocBuilder(i)));
+                numDocsEmptyIdx++;
+            }
+        }
 
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 1).setSource(
+        // randomized setup for index with empty buckets
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 1).endObject()));
+        numBuckets_empty_rnd = randomIntBetween(20, 100);
+        valueCounts_empty_rnd = new Long[numBuckets_empty_rnd];
+        firstDerivValueCounts_empty_rnd = new Double[numBuckets_empty_rnd];
+        firstDerivValueCounts_empty_rnd[0] = null;
 
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 2).setSource(
+        assertAcked(prepareCreate("empty_bucket_idx_rnd").addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=integer"));
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 2).endObject()));
+        for (int i = 0; i < numBuckets_empty_rnd; i++) {
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 3).setSource(
+            valueCounts_empty_rnd[i] = (long) randomIntBetween(1, 10);
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 2).endObject()));
+            // make approximately half of the buckets empty
-
+            if (randomBoolean())
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 4).setSource(
+                valueCounts_empty_rnd[i] = 0l;
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 4).endObject()));
+            for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) {
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 5).setSource(
+                builders.add(client().prepareIndex("empty_bucket_idx_rnd", "type").setSource(newDocBuilder(i)));
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 4).endObject()));
+                numDocsEmptyIdx_rnd++;
-
+            }
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 6).setSource(
+            if (i > 0) {
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 5).endObject()));
+                firstDerivValueCounts_empty_rnd[i] = (double) valueCounts_empty_rnd[i] - valueCounts_empty_rnd[i - 1];
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 7).setSource(
+            }
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 5).endObject()));
+        }
-
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 8).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 9).endObject()));
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 9).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 9).endObject()));
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 10).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 9).endObject()));
-
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 11).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 10).endObject()));
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 12).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 10).endObject()));
-
-        builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + 13).setSource(
-                jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, 11).endObject()));
 
         indexRandom(true, builders);
         ensureSearchable();
     }
 
-    @Test
+    private XContentBuilder newDocBuilder(int singleValueFieldValue) throws IOException {
-    public void singleValuedField() {
+        return jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, singleValueFieldValue).endObject();
-
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(
-                        histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)
-                                .subAggregation(derivative("deriv").setBucketsPaths("_count")))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        InternalHistogram deriv = response.getAggregations().get("histo");
-        assertThat(deriv, notNullValue());
-        assertThat(deriv.getName(), equalTo("histo"));
-        List<? extends Bucket> buckets = deriv.getBuckets();
-        assertThat(buckets.size(), equalTo(numValueBuckets));
-
-        for (int i = 0; i < numValueBuckets; ++i) {
-            Histogram.Bucket bucket = buckets.get(i);
-            assertThat(bucket, notNullValue());
-            assertThat(bucket.getKeyAsString(), equalTo(String.valueOf(i * interval)));
-            assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) i * interval));
-            assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
-            SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
-            if (i > 0) {
-                assertThat(docCountDeriv, notNullValue());
-                assertThat(docCountDeriv.value(), equalTo((double) firstDerivValueCounts[i - 1]));
-            } else {
-                assertThat(docCountDeriv, nullValue());
-            }
-        }
     }
 
+    /**
+     * test first and second derivative on the sing
+     */
     @Test
-    public void singleValuedField_secondDerivative() {
+    public void singleValuedField() {
 
         SearchResponse response = client()
                 .prepareSearch("idx")
@@ -216,7 +171,7 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
         assertSearchResponse(response);
 
-        InternalHistogram deriv = response.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         List<? extends Bucket> buckets = deriv.getBuckets();
@@ -224,10 +179,7 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
         for (int i = 0; i < numValueBuckets; ++i) {
             Histogram.Bucket bucket = buckets.get(i);
-            assertThat(bucket, notNullValue());
+            checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
-            assertThat(bucket.getKeyAsString(), equalTo(String.valueOf(i * interval)));
-            assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) i * interval));
-            assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
             SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
             if (i > 0) {
                 assertThat(docCountDeriv, notNullValue());
@@ -256,7 +208,7 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
         assertSearchResponse(response);
 
-        InternalHistogram deriv = response.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets));
@@ -264,92 +216,44 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
         Object[] propertiesDocCounts = (Object[]) deriv.getProperty("_count");
         Object[] propertiesSumCounts = (Object[]) deriv.getProperty("sum.value");
 
-        List<Histogram.Bucket> buckets = new ArrayList<>(deriv.getBuckets());
+        List<Bucket> buckets = new ArrayList<Bucket>(deriv.getBuckets());
+        Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets
+                                                         // overwritten
         for (int i = 0; i < numValueBuckets; ++i) {
             Histogram.Bucket bucket = buckets.get(i);
-            assertThat(bucket, notNullValue());
+            checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
-            assertThat(bucket.getKeyAsString(), equalTo(String.valueOf(i * interval)));
-            assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) i * interval));
-            assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
-            assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
             Sum sum = bucket.getAggregations().get("sum");
             assertThat(sum, notNullValue());
-            long s = 0;
+            long expectedSum = valueCounts[i] * (i * interval);
-            for (int j = 0; j < numDocs; ++j) {
+            assertThat(sum.getValue(), equalTo((double) expectedSum));
-                if ((j + 1) / interval == i) {
-                    s += j + 1;
-                }
-            }
             SimpleValue sumDeriv = bucket.getAggregations().get("deriv");
-            assertThat(sum.getValue(), equalTo((double) s));
             if (i > 0) {
                 assertThat(sumDeriv, notNullValue());
-                long s1 = 0;
+                long sumDerivValue = expectedSum - expectedSumPreviousBucket;
-                long s2 = 0;
-                for (int j = 0; j < numDocs; ++j) {
-                    if ((j + 1) / interval == i - 1) {
-                        s1 += j + 1;
-                    }
-                    if ((j + 1) / interval == i) {
-                        s2 += j + 1;
-                    }
-                }
-                long sumDerivValue = s2 - s1;
                 assertThat(sumDeriv.value(), equalTo((double) sumDerivValue));
                 assertThat((double) bucket.getProperty("histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
                         equalTo((double) sumDerivValue));
             } else {
                 assertThat(sumDeriv, nullValue());
             }
+            expectedSumPreviousBucket = expectedSum;
             assertThat((long) propertiesKeys[i], equalTo((long) i * interval));
             assertThat((long) propertiesDocCounts[i], equalTo(valueCounts[i]));
-            assertThat((double) propertiesSumCounts[i], equalTo((double) s));
+            assertThat((double) propertiesSumCounts[i], equalTo((double) expectedSum));
-        }
-    }
-
-    @Test
-    public void multiValuedField() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(
-                        histogram("histo").field(MULTI_VALUED_FIELD_NAME).interval(interval)
-                        .subAggregation(derivative("deriv").setBucketsPaths("_count")))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        InternalHistogram deriv = response.getAggregations().get("histo");
-        assertThat(deriv, notNullValue());
-        assertThat(deriv.getName(), equalTo("histo"));
-        List<? extends Bucket> buckets = deriv.getBuckets();
-        assertThat(deriv.getBuckets().size(), equalTo(numValuesBuckets));
-
-        for (int i = 0; i < numValuesBuckets; ++i) {
-            Histogram.Bucket bucket = buckets.get(i);
-            assertThat(bucket, notNullValue());
-            assertThat(bucket.getKeyAsString(), equalTo(String.valueOf(i * interval)));
-            assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) i * interval));
-            assertThat(bucket.getDocCount(), equalTo(valuesCounts[i]));
-            SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
-            if (i > 0) {
-                assertThat(docCountDeriv, notNullValue());
-                assertThat(docCountDeriv.value(), equalTo((double) firstDerivValuesCounts[i - 1]));
-            } else {
-                assertThat(docCountDeriv, nullValue());
-            }
         }
     }
 
     @Test
     public void unmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx_unmapped")
+        SearchResponse response = client()
+                .prepareSearch("idx_unmapped")
                 .addAggregation(
                         histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)
-                        .subAggregation(derivative("deriv").setBucketsPaths("_count")))
+                                .subAggregation(derivative("deriv").setBucketsPaths("_count"))).execute().actionGet();
-                .execute().actionGet();
 
         assertSearchResponse(response);
 
-        InternalHistogram deriv = response.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         assertThat(deriv.getBuckets().size(), equalTo(0));
@@ -357,15 +261,15 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void partiallyUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "idx_unmapped")
+        SearchResponse response = client()
+                .prepareSearch("idx", "idx_unmapped")
                 .addAggregation(
                         histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)
-                        .subAggregation(derivative("deriv").setBucketsPaths("_count")))
+                                .subAggregation(derivative("deriv").setBucketsPaths("_count"))).execute().actionGet();
-                .execute().actionGet();
 
         assertSearchResponse(response);
 
-        InternalHistogram deriv = response.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
         assertThat(deriv, notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
         List<? extends Bucket> buckets = deriv.getBuckets();
@@ -373,10 +277,7 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
 
         for (int i = 0; i < numValueBuckets; ++i) {
             Histogram.Bucket bucket = buckets.get(i);
-            assertThat(bucket, notNullValue());
+            checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
-            assertThat(bucket.getKeyAsString(), equalTo(String.valueOf(i * interval)));
-            assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) i * interval));
-            assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
             SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
             if (i > 0) {
                 assertThat(docCountDeriv, notNullValue());
@@ -394,111 +295,57 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).minDocCount(0)
-                                .subAggregation(derivative("deriv").setBucketsPaths("_count")))
+                                .subAggregation(derivative("deriv").setBucketsPaths("_count"))).execute().actionGet();
-                .execute().actionGet();
 
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(14l));
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocsEmptyIdx));
 
-        InternalHistogram deriv = searchResponse.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Histogram.Bucket> buckets = (List<Bucket>) deriv.getBuckets();
+        List<Bucket> buckets = deriv.getBuckets();
-        assertThat(buckets.size(), equalTo(12));
+        assertThat(buckets.size(), equalTo(valueCounts_empty.length));
 
-        Histogram.Bucket bucket = buckets.get(0);
+        for (int i = 0; i < valueCounts_empty.length; i++) {
-        assertThat(bucket, notNullValue());
+            Histogram.Bucket bucket = buckets.get(i);
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(0l));
+            checkBucketKeyAndDocCount("Bucket " + i, bucket, i, valueCounts_empty[i]);
-        assertThat(bucket.getDocCount(), equalTo(1l));
+            SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
-        SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
+            if (firstDerivValueCounts_empty[i] == null) {
-        assertThat(docCountDeriv, nullValue());
+                assertThat(docCountDeriv, nullValue());
+            } else {
+                assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty[i]));
+            }
+        }
+    }
 
-        bucket = buckets.get(1);
+    @Test
-        assertThat(bucket, notNullValue());
+    public void singleValuedFieldWithGaps_random() throws Exception {
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(1l));
+        SearchResponse searchResponse = client()
-        assertThat(bucket.getDocCount(), equalTo(1l));
+                .prepareSearch("empty_bucket_idx_rnd")
-        docCountDeriv = bucket.getAggregations().get("deriv");
+                .setQuery(matchAllQuery())
-        assertThat(docCountDeriv, notNullValue());
+                .addAggregation(
-        assertThat(docCountDeriv.value(), equalTo(0d));
+                        histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).minDocCount(0)
+                                .extendedBounds(0l, (long) numBuckets_empty_rnd - 1)
+                                .subAggregation(derivative("deriv").setBucketsPaths("_count"))).execute().actionGet();
 
-        bucket = buckets.get(2);
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocsEmptyIdx_rnd));
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(2l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(1d));
 
-        bucket = buckets.get(3);
+        InternalHistogram<Bucket> deriv = searchResponse.getAggregations().get("histo");
-        assertThat(bucket, notNullValue());
+        assertThat(deriv, Matchers.notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(3l));
+        assertThat(deriv.getName(), equalTo("histo"));
-        assertThat(bucket.getDocCount(), equalTo(0l));
+        List<Bucket> buckets = deriv.getBuckets();
-        docCountDeriv = bucket.getAggregations().get("deriv");
+        assertThat(buckets.size(), equalTo(numBuckets_empty_rnd));
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-2d));
 
-        bucket = buckets.get(4);
+        for (int i = 0; i < valueCounts_empty_rnd.length; i++) {
-        assertThat(bucket, notNullValue());
+            Histogram.Bucket bucket = buckets.get(i);
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(4l));
+            System.out.println(bucket.getDocCount());
-        assertThat(bucket.getDocCount(), equalTo(2l));
+            checkBucketKeyAndDocCount("Bucket " + i, bucket, i, valueCounts_empty_rnd[i]);
-        docCountDeriv = bucket.getAggregations().get("deriv");
+            SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
+            if (firstDerivValueCounts_empty_rnd[i] == null) {
-        assertThat(docCountDeriv.value(), equalTo(2d));
+                assertThat(docCountDeriv, nullValue());
-
+            } else {
-        bucket = buckets.get(5);
+                assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty_rnd[i]));
-        assertThat(bucket, notNullValue());
+            }
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(5l));
+        }
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(6);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(6l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-2d));
-
-        bucket = buckets.get(7);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(7l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(8);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(8l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(9);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(9l));
-        assertThat(bucket.getDocCount(), equalTo(3l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(3d));
-
-        bucket = buckets.get(10);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(10l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-1d));
-
-        bucket = buckets.get(11);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(11l));
-        assertThat(bucket.getDocCount(), equalTo(1l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-1d));
     }
 
     @Test
@@ -508,111 +355,33 @@ public class DerivativeTests extends ElasticsearchIntegrationTest {
                 .setQuery(matchAllQuery())
                 .addAggregation(
                         histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).minDocCount(0)
-                                .subAggregation(derivative("deriv").setBucketsPaths("_count").gapPolicy(BucketHelpers.GapPolicy.INSERT_ZEROS)))
+                                .subAggregation(derivative("deriv").setBucketsPaths("_count").gapPolicy(GapPolicy.INSERT_ZEROS))).execute()
-                .execute().actionGet();
+                                .actionGet();
 
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(14l));
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocsEmptyIdx));
 
-        InternalHistogram deriv = searchResponse.getAggregations().get("histo");
+        InternalHistogram<Bucket> deriv = searchResponse.getAggregations().get("histo");
         assertThat(deriv, Matchers.notNullValue());
         assertThat(deriv.getName(), equalTo("histo"));
-        List<Histogram.Bucket> buckets = (List<Bucket>) deriv.getBuckets();
+        List<Bucket> buckets = deriv.getBuckets();
-        assertThat(buckets.size(), equalTo(12));
+        assertThat(buckets.size(), equalTo(valueCounts_empty.length));
 
-        Histogram.Bucket bucket = buckets.get(0);
+        for (int i = 0; i < valueCounts_empty.length; i++) {
-        assertThat(bucket, notNullValue());
+            Histogram.Bucket bucket = buckets.get(i);
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(0l));
+            checkBucketKeyAndDocCount("Bucket " + i + ": ", bucket, i, valueCounts_empty[i]);
-        assertThat(bucket.getDocCount(), equalTo(1l));
+            SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
-        SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
+            if (firstDerivValueCounts_empty[i] == null) {
-        assertThat(docCountDeriv, nullValue());
+                assertThat(docCountDeriv, nullValue());
-
+            } else {
-        bucket = buckets.get(1);
+                assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty[i]));
-        assertThat(bucket, notNullValue());
+            }
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(1l));
+        }
-        assertThat(bucket.getDocCount(), equalTo(1l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(2l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(1d));
-
-        bucket = buckets.get(3);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(3l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-2d));
-
-        bucket = buckets.get(4);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(4l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(2d));
-
-        bucket = buckets.get(5);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(5l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(6);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(6l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-2d));
-
-        bucket = buckets.get(7);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(7l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(8);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(8l));
-        assertThat(bucket.getDocCount(), equalTo(0l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(0d));
-
-        bucket = buckets.get(9);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(9l));
-        assertThat(bucket.getDocCount(), equalTo(3l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(3d));
-
-        bucket = buckets.get(10);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(10l));
-        assertThat(bucket.getDocCount(), equalTo(2l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-1d));
-
-        bucket = buckets.get(11);
-        assertThat(bucket, notNullValue());
-        assertThat(((Number) bucket.getKey()).longValue(), equalTo(11l));
-        assertThat(bucket.getDocCount(), equalTo(1l));
-        docCountDeriv = bucket.getAggregations().get("deriv");
-        assertThat(docCountDeriv, notNullValue());
-        assertThat(docCountDeriv.value(), equalTo(-1d));
     }
 
+    private void checkBucketKeyAndDocCount(final String msg, final Histogram.Bucket bucket, final long expectedKey,
+            final long expectedDocCount) {
+        assertThat(msg, bucket, notNullValue());
+        assertThat(msg + " key", ((Number) bucket.getKey()).longValue(), equalTo(expectedKey));
+        assertThat(msg + " docCount", bucket.getDocCount(), equalTo(expectedDocCount));
+    }
 }