From 0f96a099398c3807cde1122a4d291b46b9ecca7f Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 30 Sep 2014 13:36:01 -0700 Subject: [PATCH 1/3] use multi-val dims for metrics --- .../groupby/GroupByQueryQueryToolChest.java | 16 +++++++++-- .../SegmentMetadataQueryQueryToolChest.java | 22 +++++++++++---- .../search/SearchQueryQueryToolChest.java | 23 +++++++++++----- .../select/SelectQueryQueryToolChest.java | 27 +++++++++++++------ .../TimeseriesQueryQueryToolChest.java | 17 +++++++++--- .../query/topn/TopNQueryQueryToolChest.java | 16 ++++++++--- .../server/AsyncQueryForwardingServlet.java | 11 +++++++- .../java/io/druid/server/QueryResource.java | 17 ++++++++---- 8 files changed, 115 insertions(+), 34 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index e2ddc18da5c..d27e0627085 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -218,8 +218,20 @@ public class GroupByQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser9(Minutes.minutes(numMinutes).toString()); diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index 21f02145988..d502551361c 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -22,8 +22,8 @@ package io.druid.query.metadata; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Functions; -import com.google.common.base.Joiner; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; @@ -166,7 +166,19 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser9(Minutes.minutes(numMinutes).toString()); } @@ -195,9 +207,9 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest, SearchQuery> { private static final byte SEARCH_QUERY = 0x2; - private static final Joiner COMMA_JOIN = Joiner.on(","); private static final TypeReference> TYPE_REFERENCE = new TypeReference>() { }; @@ -130,9 +127,21 @@ public class SearchQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser9(Minutes.minutes(numMinutes).toString()); } @@ -181,7 +190,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest, SelectQuery> { private static final byte SELECT_QUERY = 0x13; - private static final Joiner COMMA_JOIN = Joiner.on(","); private static final TypeReference OBJECT_TYPE_REFERENCE = new TypeReference() { @@ -133,8 +132,20 @@ public class SelectQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser9(Minutes.minutes(numMinutes).toString()); } @@ -261,13 +272,13 @@ public class SelectQueryQueryToolChest extends QueryToolChest) jsonMapper.convertValue( resultIter.next(), new TypeReference>() - { - } + { + } ), (List) jsonMapper.convertValue( resultIter.next(), new TypeReference>() - { - } + { + } ) ) ); diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java index 3ed7e84149d..b5bced221fd 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java @@ -62,7 +62,6 @@ import java.util.Map; public class TimeseriesQueryQueryToolChest extends QueryToolChest, TimeseriesQuery> { private static final byte TIMESERIES_QUERY = 0x0; - private static final Joiner COMMA_JOIN = Joiner.on(","); private static final TypeReference OBJECT_TYPE_REFERENCE = new TypeReference() { @@ -131,8 +130,20 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser9(Minutes.minutes(numMinutes).toString()); diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java index cc00ea3292e..2129c6a0bc6 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java @@ -21,7 +21,6 @@ package io.druid.query.topn; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -65,7 +64,6 @@ import java.util.Map; public class TopNQueryQueryToolChest extends QueryToolChest, TopNQuery> { private static final byte TOPN_QUERY = 0x1; - private static final Joiner COMMA_JOIN = Joiner.on(","); private static final TypeReference> TYPE_REFERENCE = new TypeReference>() { }; @@ -147,7 +145,19 @@ public class TopNQueryQueryToolChest extends QueryToolChest() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) .setUser6(String.valueOf(query.hasFilters())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser9(Minutes.minutes(numMinutes).toString()); diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index 624d52e1b36..0efb4d62ef4 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -20,7 +20,9 @@ package io.druid.server; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; @@ -41,6 +43,7 @@ import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.AsyncProxyServlet; import org.joda.time.DateTime; +import org.joda.time.Interval; import javax.servlet.AsyncContext; import javax.servlet.ServletException; @@ -259,6 +262,12 @@ public class AsyncQueryForwardingServlet extends AsyncProxyServlet public void onComplete(Result result) { final long requestTime = System.currentTimeMillis() - start; + // toArray() will give compilation error + final String[] intervals = new String[query.getIntervals().size()]; + int i = 0; + for (Object interval : query.getIntervals()) { + intervals[i] = interval.toString(); + } try { emitter.emit( new ServiceMetricEvent.Builder() @@ -271,7 +280,7 @@ public class AsyncQueryForwardingServlet extends AsyncProxyServlet ) ) .setUser4(query.getType()) - .setUser5(DataSourceUtil.COMMA_JOIN.join(query.getIntervals())) + .setUser5(intervals) .setUser6(String.valueOf(query.hasFilters())) .setUser7(req.getRemoteAddr()) .setUser8(query.getId()) diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index 7cef7893280..001ef43295e 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -23,9 +23,10 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Charsets; -import com.google.common.base.Joiner; +import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; import com.google.inject.Inject; import com.metamx.common.guava.Sequence; @@ -37,13 +38,13 @@ import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; -import io.druid.query.DataSourceUtil; import io.druid.query.Query; import io.druid.query.QueryInterruptedException; import io.druid.query.QuerySegmentWalker; import io.druid.server.initialization.ServerConfig; import io.druid.server.log.RequestLogger; import org.joda.time.DateTime; +import org.joda.time.Interval; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -59,6 +60,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.UUID; /** @@ -67,7 +69,6 @@ import java.util.UUID; public class QueryResource { private static final EmittingLogger log = new EmittingLogger(QueryResource.class); - private static final Joiner COMMA_JOIN = Joiner.on(","); public static final String APPLICATION_SMILE = "application/smile"; public static final String APPLICATION_JSON = "application/json"; @@ -175,9 +176,15 @@ public class QueryResource try { long requestTime = System.currentTimeMillis() - start; + // toArray() will give compilation error + final String[] intervals = new String[query.getIntervals().size()]; + int i = 0; + for (Object interval : query.getIntervals()) { + intervals[i] = interval.toString(); + } emitter.emit( new ServiceMetricEvent.Builder() - .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) + .setUser2(query.getDataSource().getNames().toArray(new String[query.getDataSource().getNames().size()])) .setUser3( jsonMapper.writeValueAsString( query.getContext() == null @@ -186,7 +193,7 @@ public class QueryResource ) ) .setUser4(query.getType()) - .setUser5(COMMA_JOIN.join(query.getIntervals())) + .setUser5(intervals) .setUser6(String.valueOf(query.hasFilters())) .setUser7(req.getRemoteAddr()) .setUser8(queryId) From 267e8611620f1fb5ffe6b493b388c7e307fd5a88 Mon Sep 17 00:00:00 2001 From: fjy Date: Tue, 30 Sep 2014 13:36:55 -0700 Subject: [PATCH 2/3] fix search --- .../java/io/druid/query/search/SearchQueryQueryToolChest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java index 9fb5ed99f04..4e322d231fe 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java @@ -38,6 +38,7 @@ import com.metamx.common.guava.nary.BinaryFn; import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.query.CacheStrategy; +import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; import io.druid.query.QueryRunner; @@ -127,7 +128,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest Date: Thu, 2 Oct 2014 17:51:33 -0700 Subject: [PATCH 3/3] cleanup according to cr --- .../java/io/druid/query/DataSourceUtil.java | 4 -- .../java/io/druid/query/QueryMetricUtil.java | 71 +++++++++++++++++++ .../groupby/GroupByQueryQueryToolChest.java | 33 ++------- .../SegmentMetadataQueryQueryToolChest.java | 38 +--------- .../search/SearchQueryQueryToolChest.java | 28 +------- .../select/SelectQueryQueryToolChest.java | 29 +------- .../TimeseriesQueryQueryToolChest.java | 31 +------- .../query/topn/TopNQueryQueryToolChest.java | 37 +++------- .../query/metadata/SegmentAnalyzerTest.java | 2 +- .../metadata/SegmentMetadataQueryTest.java | 2 +- .../server/AsyncQueryForwardingServlet.java | 30 +------- .../java/io/druid/server/QueryResource.java | 30 +------- 12 files changed, 102 insertions(+), 233 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/QueryMetricUtil.java diff --git a/processing/src/main/java/io/druid/query/DataSourceUtil.java b/processing/src/main/java/io/druid/query/DataSourceUtil.java index 7c056f1eabc..109db95ee04 100644 --- a/processing/src/main/java/io/druid/query/DataSourceUtil.java +++ b/processing/src/main/java/io/druid/query/DataSourceUtil.java @@ -19,14 +19,10 @@ package io.druid.query; -import com.google.common.base.Joiner; - import java.util.List; public class DataSourceUtil { - public static final Joiner COMMA_JOIN = Joiner.on(","); - public static String getMetricName(DataSource dataSource) { final List names = dataSource.getNames(); diff --git a/processing/src/main/java/io/druid/query/QueryMetricUtil.java b/processing/src/main/java/io/druid/query/QueryMetricUtil.java new file mode 100644 index 00000000000..8565d8cc7e4 --- /dev/null +++ b/processing/src/main/java/io/druid/query/QueryMetricUtil.java @@ -0,0 +1,71 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.emitter.service.ServiceMetricEvent; +import org.joda.time.Interval; + +/** + */ +public class QueryMetricUtil +{ + public static ServiceMetricEvent.Builder makeQueryTimeMetric(Query query) + { + return new ServiceMetricEvent.Builder() + .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) + .setUser4(query.getType()) + .setUser5( + Lists.transform( + query.getIntervals(), + new Function() + { + @Override + public String apply(Interval input) + { + return input.toString(); + } + } + ).toArray(new String[query.getIntervals().size()]) + ) + .setUser6(String.valueOf(query.hasFilters())) + .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()); + } + + public static ServiceMetricEvent.Builder makeRequestTimeMetric( + final ObjectMapper jsonMapper, final Query query, final String remoteAddr + ) throws JsonProcessingException + { + return makeQueryTimeMetric(query) + .setUser3( + jsonMapper.writeValueAsString( + query.getContext() == null + ? ImmutableMap.of() + : query.getContext() + ) + ) + .setUser7(remoteAddr) + .setUser8(query.getId()); + } +} diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index d27e0627085..e4931eaeb47 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -22,7 +22,6 @@ package io.druid.query.groupby; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -42,11 +41,11 @@ import io.druid.data.input.Row; import io.druid.granularity.QueryGranularity; import io.druid.query.CacheStrategy; import io.druid.query.DataSource; -import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; import io.druid.query.QueryCacheHelper; import io.druid.query.QueryDataSource; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.SubqueryQueryRunner; @@ -58,8 +57,6 @@ import io.druid.query.filter.DimFilter; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexStorageAdapter; import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Minutes; import java.nio.ByteBuffer; import java.util.Iterator; @@ -210,31 +207,9 @@ public class GroupByQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query) + .setUser3(String.format("%,d dims", query.getDimensions().size())) + .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())); } @Override diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index d502551361c..c35d49fd645 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -23,11 +23,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Functions; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; @@ -36,9 +34,8 @@ import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.common.utils.JodaUtils; import io.druid.query.CacheStrategy; -import io.druid.query.DataSourceUtil; import io.druid.query.Query; -import io.druid.query.QueryConfig; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.ResultMergeQueryRunner; @@ -47,7 +44,6 @@ import io.druid.query.metadata.metadata.ColumnAnalysis; import io.druid.query.metadata.metadata.SegmentAnalysis; import io.druid.query.metadata.metadata.SegmentMetadataQuery; import org.joda.time.Interval; -import org.joda.time.Minutes; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -62,14 +58,6 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest mergeResults(final QueryRunner runner) { @@ -158,29 +146,7 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query); } @Override diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java index 4e322d231fe..163e244223e 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java @@ -38,9 +38,9 @@ import com.metamx.common.guava.nary.BinaryFn; import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.query.CacheStrategy; -import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.Result; @@ -52,8 +52,6 @@ import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Minutes; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -122,29 +120,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query); } @Override diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index 5e7bbee4fa0..f214d2441b1 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.base.Function; import com.google.common.base.Functions; -import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.inject.Inject; @@ -35,10 +34,10 @@ import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.granularity.QueryGranularity; import io.druid.query.CacheStrategy; -import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; import io.druid.query.QueryConfig; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.Result; @@ -47,8 +46,6 @@ import io.druid.query.ResultMergeQueryRunner; import io.druid.query.aggregation.MetricManipulationFn; import io.druid.query.filter.DimFilter; import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Minutes; import java.nio.ByteBuffer; import java.util.Arrays; @@ -125,29 +122,7 @@ public class SelectQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query); } @Override diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java index b5bced221fd..31d71bc07aa 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java @@ -21,7 +21,6 @@ package io.druid.query.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; @@ -33,11 +32,11 @@ import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.granularity.QueryGranularity; import io.druid.query.CacheStrategy; -import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; import io.druid.query.QueryCacheHelper; import io.druid.query.QueryConfig; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.Result; @@ -48,8 +47,6 @@ import io.druid.query.aggregation.MetricManipulationFn; import io.druid.query.aggregation.PostAggregator; import io.druid.query.filter.DimFilter; import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Minutes; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -123,30 +120,8 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query) + .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())); } @Override diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java index 2129c6a0bc6..fe55cd7d888 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java @@ -36,10 +36,10 @@ import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.granularity.QueryGranularity; import io.druid.query.CacheStrategy; -import io.druid.query.DataSourceUtil; import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.Query; import io.druid.query.QueryCacheHelper; +import io.druid.query.QueryMetricUtil; import io.druid.query.QueryRunner; import io.druid.query.QueryToolChest; import io.druid.query.Result; @@ -51,8 +51,6 @@ import io.druid.query.aggregation.MetricManipulationFn; import io.druid.query.aggregation.PostAggregator; import io.druid.query.filter.DimFilter; import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Minutes; import java.nio.ByteBuffer; import java.util.Iterator; @@ -137,30 +135,15 @@ public class TopNQueryQueryToolChest extends QueryToolChest() - { - @Override - public String apply(Interval input) - { - return input.toString(); - } - } - ).toArray(new String[query.getIntervals().size()]) - ) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) - .setUser9(Minutes.minutes(numMinutes).toString()); + return QueryMetricUtil.makeQueryTimeMetric(query) + .setUser4( + String.format( + "topN/%s/%s", + query.getThreshold(), + query.getDimensionSpec().getDimension() + ) + ) + .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())); } @Override diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java index b9834c106cc..6130b318757 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java @@ -98,7 +98,7 @@ public class SegmentAnalyzerTest { final QueryRunner runner = QueryRunnerTestHelper.makeQueryRunner( (QueryRunnerFactory) new SegmentMetadataQueryRunnerFactory( - new SegmentMetadataQueryQueryToolChest(new QueryConfig()), + new SegmentMetadataQueryQueryToolChest(), QueryRunnerTestHelper.NOOP_QUERYWATCHER ), index ); diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index 0c58576c1d4..70336573432 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -47,7 +47,7 @@ public class SegmentMetadataQueryTest @SuppressWarnings("unchecked") private final QueryRunner runner = makeQueryRunner( new SegmentMetadataQueryRunnerFactory( - new SegmentMetadataQueryQueryToolChest(new QueryConfig()), + new SegmentMetadataQueryQueryToolChest(), QueryRunnerTestHelper.NOOP_QUERYWATCHER) ); private ObjectMapper mapper = new DefaultObjectMapper(); diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index 0efb4d62ef4..c5f6a5c7214 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -20,16 +20,13 @@ package io.druid.server; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; -import io.druid.query.DataSourceUtil; import io.druid.query.Query; +import io.druid.query.QueryMetricUtil; import io.druid.server.log.RequestLogger; import io.druid.server.router.QueryHostFinder; import io.druid.server.router.Router; @@ -43,7 +40,6 @@ import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.AsyncProxyServlet; import org.joda.time.DateTime; -import org.joda.time.Interval; import javax.servlet.AsyncContext; import javax.servlet.ServletException; @@ -262,30 +258,10 @@ public class AsyncQueryForwardingServlet extends AsyncProxyServlet public void onComplete(Result result) { final long requestTime = System.currentTimeMillis() - start; - // toArray() will give compilation error - final String[] intervals = new String[query.getIntervals().size()]; - int i = 0; - for (Object interval : query.getIntervals()) { - intervals[i] = interval.toString(); - } try { emitter.emit( - new ServiceMetricEvent.Builder() - .setUser2(DataSourceUtil.getMetricName(query.getDataSource())) - .setUser3( - jsonMapper.writeValueAsString( - query.getContext() == null - ? ImmutableMap.of() - : query.getContext() - ) - ) - .setUser4(query.getType()) - .setUser5(intervals) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(req.getRemoteAddr()) - .setUser8(query.getId()) - .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()) - .build("request/time", requestTime) + QueryMetricUtil.makeRequestTimeMetric(jsonMapper, query, req.getRemoteAddr()) + .build("request/time", requestTime) ); requestLogger.log( diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index 001ef43295e..0ce414f1b8e 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -23,10 +23,8 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Charsets; -import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; import com.google.inject.Inject; import com.metamx.common.guava.Sequence; @@ -35,16 +33,15 @@ import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; import io.druid.query.Query; import io.druid.query.QueryInterruptedException; +import io.druid.query.QueryMetricUtil; import io.druid.query.QuerySegmentWalker; import io.druid.server.initialization.ServerConfig; import io.druid.server.log.RequestLogger; import org.joda.time.DateTime; -import org.joda.time.Interval; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -60,7 +57,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.IOException; import java.io.OutputStream; -import java.util.List; import java.util.UUID; /** @@ -176,29 +172,9 @@ public class QueryResource try { long requestTime = System.currentTimeMillis() - start; - // toArray() will give compilation error - final String[] intervals = new String[query.getIntervals().size()]; - int i = 0; - for (Object interval : query.getIntervals()) { - intervals[i] = interval.toString(); - } emitter.emit( - new ServiceMetricEvent.Builder() - .setUser2(query.getDataSource().getNames().toArray(new String[query.getDataSource().getNames().size()])) - .setUser3( - jsonMapper.writeValueAsString( - query.getContext() == null - ? ImmutableMap.of() - : query.getContext() - ) - ) - .setUser4(query.getType()) - .setUser5(intervals) - .setUser6(String.valueOf(query.hasFilters())) - .setUser7(req.getRemoteAddr()) - .setUser8(queryId) - .setUser9(query.getDuration().toPeriod().toStandardMinutes().toString()) - .build("request/time", requestTime) + QueryMetricUtil.makeRequestTimeMetric(jsonMapper, query, req.getRemoteAddr()) + .build("request/time", requestTime) ); requestLogger.log(