From 771a8893a669fd38a586ffdf47ed1de232d422e6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Sep 2020 13:21:11 -0400 Subject: [PATCH] Add more debugging information for cardinality agg (#62317) (#62397) This adds two extra bits of info to the profiler: 1. Count of the number of different types of collectors. This lets us figure out if we're using the optimization for segment ordinals. It adds a few more similar counters just for good measure. 2. Profiles the `getLeafCollector` and `postCollection` methods. These are non-trivial for some aggregations, like cardinality. --- docs/reference/search/profile.asciidoc | 18 ++++- .../170_cardinality_metric.yml | 78 +++++++++++++++++++ .../aggregation/AggregationProfilerIT.java | 46 +++++++++-- .../metrics/CardinalityAggregator.java | 36 +++++++-- .../aggregation/AggregationTimingType.java | 2 + .../aggregation/ProfilingAggregator.java | 16 +++- .../runtime-fields/qa/rest/build.gradle | 1 + 7 files changed, 180 insertions(+), 17 deletions(-) diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index de61ff8b8e7..a431c48414c 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -794,7 +794,11 @@ This yields the following aggregation profile output: "initialize_count": 1, "reduce_count": 0, "collect": 45786, - "collect_count": 4 + "collect_count": 4, + "build_leaf_collector": 18211, + "build_leaf_collector_count": 1, + "post_collection": 929, + "post_collection_count": 1 }, "debug": { "total_buckets": 1, @@ -813,7 +817,11 @@ This yields the following aggregation profile output: "initialize_count": 1, "reduce_count": 0, "collect": 69401, - "collect_count": 4 + "collect_count": 4, + "build_leaf_collector": 8150, + "build_leaf_collector_count": 1, + "post_collection": 1584, + "post_collection_count": 1 }, "children": [ { @@ -828,7 +836,11 @@ This yields the following aggregation profile output: "initialize_count": 1, "reduce_count": 0, "collect": 61611, - "collect_count": 4 + "collect_count": 4, + "build_leaf_collector": 5564, + "build_leaf_collector_count": 1, + "post_collection": 471, + "post_collection_count": 1 }, "debug": { "total_buckets": 1, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml index 482ab05291a..d0bd46a8090 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml @@ -212,3 +212,81 @@ setup: cardinality: field: int_field precision_threshold: -1 + +--- +"profiler int": + - skip: + version: " - 7.99.99" + reason: new info added in 8.0.0 to be backported to 7.10.0 + - do: + search: + body: + profile: true + size: 0 + aggs: + distinct_int: + cardinality: + field: int_field + - match: { aggregations.distinct_int.value: 4 } + - gt: { profile.shards.0.aggregations.0.breakdown.initialize: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_leaf_collector: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.collect: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_aggregation: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.post_collection: 0 } + - match: { profile.shards.0.aggregations.0.debug.empty_collectors_used: 0 } + - gt: { profile.shards.0.aggregations.0.debug.numeric_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.ordinals_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.ordinals_collectors_overhead_too_high: 0 } + - match: { profile.shards.0.aggregations.0.debug.string_hashing_collectors_used: 0 } + +--- +"profiler double": + - skip: + version: " - 7.99.99" + reason: new info added in 8.0.0 to be backported to 7.10.0 + - do: + search: + body: + profile: true + size: 0 + aggs: + distinct_double: + cardinality: + field: double_field + - match: { aggregations.distinct_double.value: 4 } + - gt: { profile.shards.0.aggregations.0.breakdown.initialize: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_leaf_collector: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.collect: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_aggregation: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.post_collection: 0 } + - match: { profile.shards.0.aggregations.0.debug.empty_collectors_used: 0 } + - gt: { profile.shards.0.aggregations.0.debug.numeric_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.ordinals_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.ordinals_collectors_overhead_too_high: 0 } + - match: { profile.shards.0.aggregations.0.debug.string_hashing_collectors_used: 0 } + +--- +"profiler string": + - skip: + version: " - 7.99.99" + reason: new info added in 8.0.0 to be backported to 7.10.0 + - do: + search: + body: + profile: true + size: 0 + aggs: + distinct_string: + cardinality: + field: string_field + - match: { aggregations.distinct_string.value: 1 } + - gt: { profile.shards.0.aggregations.0.breakdown.initialize: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_leaf_collector: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.collect: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.build_aggregation: 0 } + - gt: { profile.shards.0.aggregations.0.breakdown.post_collection: 0 } + - match: { profile.shards.0.aggregations.0.debug.empty_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.numeric_collectors_used: 0 } + - gt: { profile.shards.0.aggregations.0.debug.ordinals_collectors_used: 0 } + - match: { profile.shards.0.aggregations.0.debug.ordinals_collectors_overhead_too_high: 0 } + - match: { profile.shards.0.aggregations.0.debug.string_hashing_collectors_used: 0 } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java index 6221769d5a6..f4dda12057f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java @@ -50,16 +50,28 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.SuiteScopeTestCase public class AggregationProfilerIT extends ESIntegTestCase { + private static final String BUILD_LEAF_COLLECTOR = AggregationTimingType.BUILD_LEAF_COLLECTOR.toString(); private static final String COLLECT = AggregationTimingType.COLLECT.toString(); + private static final String POST_COLLECTION = AggregationTimingType.POST_COLLECTION.toString(); private static final String INITIALIZE = AggregationTimingType.INITIALIZE.toString(); private static final String BUILD_AGGREGATION = AggregationTimingType.BUILD_AGGREGATION.toString(); private static final String REDUCE = AggregationTimingType.REDUCE.toString(); private static final Set BREAKDOWN_KEYS = org.elasticsearch.common.collect.Set.of( - COLLECT, INITIALIZE, BUILD_AGGREGATION, REDUCE, - COLLECT + "_count", INITIALIZE + "_count", BUILD_AGGREGATION + "_count", REDUCE + "_count"); + INITIALIZE, + BUILD_LEAF_COLLECTOR, + COLLECT, + POST_COLLECTION, + BUILD_AGGREGATION, + REDUCE, + INITIALIZE + "_count", + BUILD_LEAF_COLLECTOR + "_count", + COLLECT + "_count", + POST_COLLECTION + "_count", + BUILD_AGGREGATION + "_count", + REDUCE + "_count" + ); private static final String TOTAL_BUCKETS = "total_buckets"; - private static final String WRAPPED = "wrapped_in_multi_bucket_aggregator"; private static final String DEFERRED = "deferred_aggregators"; private static final String COLLECTION_STRAT = "collection_strategy"; private static final String RESULT_STRAT = "result_strategy"; @@ -316,7 +328,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(diversifyBreakdown, notNullValue()); assertThat(diversifyBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(diversifyBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(diversifyBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(diversifyBreakdown.get(REDUCE), equalTo(0L)); assertThat(diversifyAggResult.getDebugInfo(), equalTo( @@ -331,8 +345,10 @@ public class AggregationProfilerIT extends ESIntegTestCase { Map maxBreakdown = maxAggResult.getTimeBreakdown(); assertThat(maxBreakdown, notNullValue()); assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); - assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); - assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); + assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(diversifyBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(maxBreakdown.get(REDUCE), equalTo(0L)); assertThat(maxAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -374,7 +390,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(histoBreakdown, notNullValue()); assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(histoBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(histoBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(histoBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(histoBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(histoBreakdown.get(REDUCE), equalTo(0L)); Map histoDebugInfo = histoAggResult.getDebugInfo(); @@ -394,7 +412,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(tagsBreakdown, notNullValue()); assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(tagsBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(tagsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(tagsBreakdown.get(REDUCE), equalTo(0L)); assertRemapTermsDebugInfo(tagsAggResult); @@ -411,7 +431,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(avgBreakdown, notNullValue()); assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(avgBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(avgBreakdown.get(REDUCE), equalTo(0L)); assertThat(avgAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -425,7 +447,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(maxBreakdown, notNullValue()); assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(maxBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(maxBreakdown.get(REDUCE), equalTo(0L)); assertThat(maxAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -439,7 +463,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(stringsBreakdown, notNullValue()); assertThat(stringsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(stringsBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(stringsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(stringsBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(stringsBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(stringsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(stringsBreakdown.get(REDUCE), equalTo(0L)); assertRemapTermsDebugInfo(stringsAggResult); @@ -456,7 +482,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(avgBreakdown, notNullValue()); assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(avgBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(avgBreakdown.get(REDUCE), equalTo(0L)); assertThat(avgAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -470,7 +498,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(maxBreakdown, notNullValue()); assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(maxBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(maxBreakdown.get(REDUCE), equalTo(0L)); assertThat(maxAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -485,7 +515,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(tagsBreakdown, notNullValue()); assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(tagsBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(tagsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(tagsBreakdown.get(REDUCE), equalTo(0L)); assertRemapTermsDebugInfo(tagsAggResult); @@ -502,7 +534,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(avgBreakdown, notNullValue()); assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(avgBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(avgBreakdown.get(REDUCE), equalTo(0L)); assertThat(avgAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); @@ -516,7 +550,9 @@ public class AggregationProfilerIT extends ESIntegTestCase { assertThat(maxBreakdown, notNullValue()); assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); + assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); + assertThat(maxBreakdown.get(POST_COLLECTION), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); assertThat(maxBreakdown.get(REDUCE), equalTo(0L)); assertThat(maxAggResult.getDebugInfo(), equalTo(org.elasticsearch.common.collect.Map.of())); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java index 50315b9f939..c617dafb840 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregator.java @@ -30,6 +30,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; @@ -46,6 +47,7 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.Map; +import java.util.function.BiConsumer; /** * An aggregator that computes approximate counts of unique values. @@ -61,6 +63,12 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private Collector collector; + private int emptyCollectorsUsed; + private int numericCollectorsUsed; + private int ordinalsCollectorsUsed; + private int ordinalsCollectorsOverheadTooHigh; + private int stringHashingCollectorsUsed; + public CardinalityAggregator( String name, ValuesSourceConfig valuesSourceConfig, @@ -82,6 +90,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource == null) { + emptyCollectorsUsed++; return new EmptyCollector(); } @@ -89,6 +98,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; MurmurHash3Values hashValues = source.isFloatingPoint() ? MurmurHash3Values.hash(source.doubleValues(ctx)) : MurmurHash3Values.hash(source.longValues(ctx)); + numericCollectorsUsed++; return new DirectCollector(counts, hashValues); } @@ -97,6 +107,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue final SortedSetDocValues ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { + emptyCollectorsUsed++; return new EmptyCollector(); } @@ -104,10 +115,13 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue final long countsMemoryUsage = HyperLogLogPlusPlus.memoryUsage(precision); // only use ordinals if they don't increase memory usage by more than 25% if (ordinalsMemoryUsage < countsMemoryUsage / 4) { + ordinalsCollectorsUsed++; return new OrdinalsCollector(counts, ordinalValues, context.bigArrays()); } + ordinalsCollectorsOverheadTooHigh++; } + stringHashingCollectorsUsed++; return new DirectCollector(counts, MurmurHash3Values.hash(valuesSource.bytesValues(ctx))); } @@ -163,6 +177,16 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue Releasables.close(counts, collector); } + @Override + public void collectDebugInfo(BiConsumer add) { + super.collectDebugInfo(add); + add.accept("empty_collectors_used", emptyCollectorsUsed); + add.accept("numeric_collectors_used", numericCollectorsUsed); + add.accept("ordinals_collectors_used", ordinalsCollectorsUsed); + add.accept("ordinals_collectors_overhead_too_high", ordinalsCollectorsOverheadTooHigh); + add.accept("string_hashing_collectors_used", stringHashingCollectorsUsed); + } + private abstract static class Collector extends LeafBucketCollector implements Releasable { public abstract void postCollect() throws IOException; @@ -234,7 +258,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private final SortedSetDocValues values; private final int maxOrd; private final HyperLogLogPlusPlus counts; - private ObjectArray visitedOrds; + private ObjectArray visitedOrds; // Danger! This is not tracked by BigArrays! OrdinalsCollector(HyperLogLogPlusPlus counts, SortedSetDocValues values, BigArrays bigArrays) { @@ -273,12 +297,12 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue } } - final org.elasticsearch.common.hash.MurmurHash3.Hash128 hash = new org.elasticsearch.common.hash.MurmurHash3.Hash128(); + final MurmurHash3.Hash128 hash = new MurmurHash3.Hash128(); try (LongArray hashes = bigArrays.newLongArray(maxOrd, false)) { for (int ord = allVisitedOrds.nextSetBit(0); ord < DocIdSetIterator.NO_MORE_DOCS; ord = ord + 1 < maxOrd ? allVisitedOrds.nextSetBit(ord + 1) : DocIdSetIterator.NO_MORE_DOCS) { final BytesRef value = values.lookupOrd(ord); - org.elasticsearch.common.hash.MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash); + MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash); hashes.set(ord, hash.h1); } @@ -298,7 +322,6 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue public void close() { Releasables.close(visitedOrds); } - } /** @@ -383,8 +406,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private static class Bytes extends MurmurHash3Values { - private final org.elasticsearch.common.hash.MurmurHash3.Hash128 hash = - new org.elasticsearch.common.hash.MurmurHash3.Hash128(); + private final MurmurHash3.Hash128 hash = new MurmurHash3.Hash128(); private final SortedBinaryDocValues values; @@ -405,7 +427,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue @Override public long nextValue() throws IOException { final BytesRef bytes = values.nextValue(); - org.elasticsearch.common.hash.MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, hash); + MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, hash); return hash.h1; } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java index d1c5d3dd538..8a65ef2f72c 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java +++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java @@ -23,7 +23,9 @@ import java.util.Locale; public enum AggregationTimingType { INITIALIZE, + BUILD_LEAF_COLLECTOR, COLLECT, + POST_COLLECTION, BUILD_AGGREGATION, REDUCE; diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java index 97e1a366d79..1944904dfd1 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java @@ -102,7 +102,13 @@ public class ProfilingAggregator extends Aggregator { @Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException { - return new ProfilingLeafBucketCollector(delegate.getLeafCollector(ctx), profileBreakdown); + Timer timer = profileBreakdown.getTimer(AggregationTimingType.BUILD_LEAF_COLLECTOR); + timer.start(); + try { + return new ProfilingLeafBucketCollector(delegate.getLeafCollector(ctx), profileBreakdown); + } finally { + timer.stop(); + } } @Override @@ -120,7 +126,13 @@ public class ProfilingAggregator extends Aggregator { @Override public void postCollection() throws IOException { - delegate.postCollection(); + Timer timer = profileBreakdown.getTimer(AggregationTimingType.POST_COLLECTION); + timer.start(); + try { + delegate.postCollection(); + } finally { + timer.stop(); + } } @Override diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle index 59fad2f2f2d..cd934ed7b0a 100644 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/rest/build.gradle @@ -46,6 +46,7 @@ yamlRestTest { // Runtime fields don't have global ords 'search.aggregation/20_terms/string profiler via global ordinals', 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', //dynamic template causes a type _doc to be created, these tests use another type but only one type is allowed 'search.aggregation/51_filter_with_types/*', 'search/171_terms_query_with_types/*',