From 2c12c3e628dfe26c9d752fad902d06cf9b613fdc Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Mon, 25 Jul 2016 10:19:15 +0100 Subject: [PATCH] Add _bucket_count option to buckets_path This change adds a new special path to the buckets_path syntax `_bucket_count`. This new option will return the number of buckets for a multi-bucket aggregation, which can then be used in pipeline aggregations. Closes #19553 --- .../InternalMultiBucketAggregation.java | 6 ++- .../aggregations/bucket/DateHistogramIT.java | 3 +- .../aggregations/bucket/DateRangeIT.java | 5 ++- .../aggregations/bucket/DoubleTermsIT.java | 1 + .../search/aggregations/bucket/FiltersIT.java | 2 + .../aggregations/bucket/GeoDistanceIT.java | 3 +- .../aggregations/bucket/HistogramIT.java | 22 +++++----- docs/reference/aggregations/pipeline.asciidoc | 41 ++++++++++++++++++- 8 files changed, 64 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java index f1e8b7358c0..a1326aaed11 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java @@ -43,7 +43,7 @@ public abstract class InternalMultiBucketAggregation path) { if (path.isEmpty()) { return this; + } else if (path.get(0).equals("_bucket_count")) { + return getBuckets().size(); } else { List buckets = getBuckets(); Object[] propertyArray = new Object[buckets.size()]; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 2c3534183e0..ace55c2c616 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -380,6 +380,7 @@ public class DateHistogramIT extends ESIntegTestCase { assertThat(histo.getName(), equalTo("histo")); List buckets = histo.getBuckets(); assertThat(buckets.size(), equalTo(3)); + assertThat(histo.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) histo.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count"); Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value"); @@ -600,7 +601,7 @@ public class DateHistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(4)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); Histogram.Bucket bucket = buckets.get(0); assertThat(bucket, notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 36613cfa784..1e250681004 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -137,7 +137,7 @@ public class DateRangeIT extends ESIntegTestCase { assertThat(range.getBuckets().size(), equalTo(3)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(range.getBuckets()); + List buckets = new ArrayList<>(range.getBuckets()); Range.Bucket bucket = buckets.get(0); assertThat((String) bucket.getKey(), equalTo("a long time ago")); @@ -421,6 +421,7 @@ public class DateRangeIT extends ESIntegTestCase { assertThat(range.getName(), equalTo("range")); List buckets = range.getBuckets(); assertThat(buckets.size(), equalTo(3)); + assertThat(range.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) range.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) range.getProperty("_count"); Object[] propertiesCounts = (Object[]) range.getProperty("sum.value"); @@ -855,7 +856,7 @@ public class DateRangeIT extends ESIntegTestCase { Range dateRange = bucket.getAggregations().get("date_range"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(dateRange.getBuckets()); + List buckets = new ArrayList<>(dateRange.getBuckets()); assertThat(dateRange, Matchers.notNullValue()); assertThat(dateRange.getName(), equalTo("date_range")); assertThat(buckets.size(), is(1)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index 6bcb07ef9d8..ed43ba8bd63 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -426,6 +426,7 @@ public class DoubleTermsIT extends AbstractTermsTestCase { assertThat(terms, notNullValue()); assertThat(terms.getName(), equalTo("terms")); assertThat(terms.getBuckets().size(), equalTo(5)); + assertThat(terms.getProperty("_bucket_count"), equalTo(5)); Object[] propertiesKeys = (Object[]) terms.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) terms.getProperty("_count"); Object[] propertiesCounts = (Object[]) terms.getProperty("sum.value"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index 592861ccce2..619bba7da12 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -172,6 +172,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(filters.getName(), equalTo("tags")); assertThat(filters.getBuckets().size(), equalTo(2)); + assertThat(filters.getProperty("_bucket_count"), equalTo(2)); Object[] propertiesKeys = (Object[]) filters.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count"); Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value"); @@ -426,6 +427,7 @@ public class FiltersIT extends ESIntegTestCase { assertThat(filters.getName(), equalTo("tags")); assertThat(filters.getBuckets().size(), equalTo(3)); + assertThat(filters.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) filters.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) filters.getProperty("_count"); Object[] propertiesCounts = (Object[]) filters.getProperty("avg_value.value"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index a27b193f435..a69513af77d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -349,6 +349,7 @@ public class GeoDistanceIT extends ESIntegTestCase { assertThat(geoDist.getName(), equalTo("amsterdam_rings")); List buckets = geoDist.getBuckets(); assertThat(geoDist.getBuckets().size(), equalTo(3)); + assertThat(geoDist.getProperty("_bucket_count"), equalTo(3)); Object[] propertiesKeys = (Object[]) geoDist.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) geoDist.getProperty("_count"); Object[] propertiesCities = (Object[]) geoDist.getProperty("cities"); @@ -429,7 +430,7 @@ public class GeoDistanceIT extends ESIntegTestCase { Range geoDistance = bucket.getAggregations().get("geo_dist"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(geoDistance.getBuckets()); + List buckets = new ArrayList<>(geoDistance.getBuckets()); assertThat(geoDistance, Matchers.notNullValue()); assertThat(geoDistance.getName(), equalTo("geo_dist")); assertThat(buckets.size(), is(1)); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index dc439318978..8df4a8ca608 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -253,7 +252,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -276,7 +275,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(numValueBuckets - i - 1); assertThat(bucket, notNullValue()); @@ -300,7 +299,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet buckets = new LongHashSet(); // TODO: use diamond once JI-9019884 is fixed - List histoBuckets = new ArrayList(histo.getBuckets()); + List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MIN_VALUE; for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = histoBuckets.get(i); @@ -329,7 +328,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet buckets = new LongHashSet(); // TODO: use diamond once JI-9019884 is fixed - List histoBuckets = new ArrayList(histo.getBuckets()); + List histoBuckets = new ArrayList<>(histo.getBuckets()); long previousCount = Long.MAX_VALUE; for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = histoBuckets.get(i); @@ -356,12 +355,13 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo, notNullValue()); assertThat(histo.getName(), equalTo("histo")); assertThat(histo.getBuckets().size(), equalTo(numValueBuckets)); + assertThat(histo.getProperty("_bucket_count"), equalTo(numValueBuckets)); Object[] propertiesKeys = (Object[]) histo.getProperty("_key"); Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count"); Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value"); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -404,7 +404,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.NEGATIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -448,7 +448,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -492,7 +492,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double previousSum = Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -538,7 +538,7 @@ public class HistogramIT extends ESIntegTestCase { LongHashSet visited = new LongHashSet(); double prevMax = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValueBuckets; ++i) { Histogram.Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); @@ -625,7 +625,7 @@ public class HistogramIT extends ESIntegTestCase { assertThat(histo.getBuckets().size(), equalTo(numValuesBuckets)); // TODO: use diamond once JI-9019884 is fixed - List buckets = new ArrayList(histo.getBuckets()); + List buckets = new ArrayList<>(histo.getBuckets()); for (int i = 0; i < numValuesBuckets; ++i) { Histogram.Bucket bucket = buckets.get(numValuesBuckets - i - 1); assertThat(bucket, notNullValue()); diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index 7ed01d9dc7f..4382596159c 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -107,8 +107,7 @@ a metric embedded inside a sibling aggregation: === Special Paths Instead of pathing to a metric, `buckets_path` can use a special `"_count"` path. This instructs -the pipeline aggregation to use the document count as it's input. For example, a moving average can be calculated on the document -count of each bucket, instead of a specific metric: +the pipeline aggregation to use the document count as it's input. For example, a moving average can be calculated on the document count of each bucket, instead of a specific metric: [source,js] -------------------------------------------------- @@ -128,6 +127,44 @@ count of each bucket, instead of a specific metric: -------------------------------------------------- <1> By using `_count` instead of a metric name, we can calculate the moving average of document counts in the histogram +The `buckets_path` can also use `"_bucket_count"` and path to a multi-bucket aggregation to use the number of buckets +returned by that aggregation in the pipeline aggregation instead of a metric. for example a `bucket_selector` can be +used here to filter out buckets which contain no buckets for an inner terms aggregation: + +[source,js] +-------------------------------------------------- +{ + "size": 0, + "aggs": { + "histo": { + "date_histogram": { + "field": "date", + "interval": "day" + }, + "aggs": { + "categories": { + "terms": { + "field": "category" + } + }, + "min_bucket_selector": { + "bucket_selector": { + "buckets_path": { + "count": "categories._bucket_count" + }, + "script": { + "inline": "count != 0" + } + } + } + } + } + } +} +-------------------------------------------------- +<1> By using `_bucket_count` instead of a metric name, we can filter out `histo` buckets where they contain no buckets +for the `categories` aggregation + [[dots-in-agg-names]] [float] === Dealing with dots in agg names