use multi-val dims for metrics

This commit is contained in:
fjy 2014-09-30 13:36:01 -07:00
parent 0781781b99
commit 0f96a09939
8 changed files with 115 additions and 34 deletions

View File

@ -218,8 +218,20 @@ public class GroupByQueryQueryToolChest extends QueryToolChest<Row, GroupByQuery
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(DataSourceUtil.getMetricName(query.getDataSource()))
.setUser3(String.format("%,d dims", query.getDimensions().size())) .setUser3(String.format("%,d dims", query.getDimensions().size()))
.setUser4("groupBy") .setUser4(query.getType())
.setUser5(Joiner.on(",").join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());

View File

@ -22,8 +22,8 @@ package io.druid.query.metadata;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Functions; import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
@ -166,7 +166,19 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest<SegmentAn
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(DataSourceUtil.getMetricName(query.getDataSource()))
.setUser4(query.getType()) .setUser4(query.getType())
.setUser5(Joiner.on(",").join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());
} }
@ -195,9 +207,9 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest<SegmentAn
{ {
byte[] includerBytes = query.getToInclude().getCacheKey(); byte[] includerBytes = query.getToInclude().getCacheKey();
return ByteBuffer.allocate(1 + includerBytes.length) return ByteBuffer.allocate(1 + includerBytes.length)
.put(SEGMENT_METADATA_CACHE_PREFIX) .put(SEGMENT_METADATA_CACHE_PREFIX)
.put(includerBytes) .put(includerBytes)
.array(); .array();
} }
@Override @Override

View File

@ -23,7 +23,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Charsets; import com.google.common.base.Charsets;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Functions; import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
@ -39,7 +38,6 @@ import com.metamx.common.guava.nary.BinaryFn;
import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.emitter.service.ServiceMetricEvent;
import io.druid.collections.OrderedMergeSequence; import io.druid.collections.OrderedMergeSequence;
import io.druid.query.CacheStrategy; import io.druid.query.CacheStrategy;
import io.druid.query.DataSourceUtil;
import io.druid.query.IntervalChunkingQueryRunner; import io.druid.query.IntervalChunkingQueryRunner;
import io.druid.query.Query; import io.druid.query.Query;
import io.druid.query.QueryRunner; import io.druid.query.QueryRunner;
@ -67,7 +65,6 @@ import java.util.Set;
public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResultValue>, SearchQuery> public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResultValue>, SearchQuery>
{ {
private static final byte SEARCH_QUERY = 0x2; private static final byte SEARCH_QUERY = 0x2;
private static final Joiner COMMA_JOIN = Joiner.on(",");
private static final TypeReference<Result<SearchResultValue>> TYPE_REFERENCE = new TypeReference<Result<SearchResultValue>>() private static final TypeReference<Result<SearchResultValue>> TYPE_REFERENCE = new TypeReference<Result<SearchResultValue>>()
{ {
}; };
@ -130,9 +127,21 @@ public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResul
} }
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(query.getDataSource().getNames().toArray(new String[query.getDataSource().getNames().size()]))
.setUser4("search") .setUser4(query.getType())
.setUser5(COMMA_JOIN.join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());
} }
@ -181,7 +190,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResul
final ByteBuffer queryCacheKey = ByteBuffer final ByteBuffer queryCacheKey = ByteBuffer
.allocate( .allocate(
1 + 4 + granularityBytes.length + filterBytes.length + 1 + 4 + granularityBytes.length + filterBytes.length +
querySpecBytes.length + dimensionsBytesSize querySpecBytes.length + dimensionsBytesSize
) )
.put(SEARCH_QUERY) .put(SEARCH_QUERY)
.put(Ints.toByteArray(query.getLimit())) .put(Ints.toByteArray(query.getLimit()))

View File

@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets; import com.google.common.base.Charsets;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Functions; import com.google.common.base.Functions;
import com.google.common.base.Joiner; import com.google.common.collect.Lists;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.google.inject.Inject; import com.google.inject.Inject;
@ -62,7 +62,6 @@ import java.util.Set;
public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResultValue>, SelectQuery> public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResultValue>, SelectQuery>
{ {
private static final byte SELECT_QUERY = 0x13; private static final byte SELECT_QUERY = 0x13;
private static final Joiner COMMA_JOIN = Joiner.on(",");
private static final TypeReference<Object> OBJECT_TYPE_REFERENCE = private static final TypeReference<Object> OBJECT_TYPE_REFERENCE =
new TypeReference<Object>() new TypeReference<Object>()
{ {
@ -133,8 +132,20 @@ public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResul
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(DataSourceUtil.getMetricName(query.getDataSource()))
.setUser4("Select") .setUser4(query.getType())
.setUser5(COMMA_JOIN.join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());
} }
@ -261,13 +272,13 @@ public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResul
new SelectResultValue( new SelectResultValue(
(Map<String, Integer>) jsonMapper.convertValue( (Map<String, Integer>) jsonMapper.convertValue(
resultIter.next(), new TypeReference<Map<String, Integer>>() resultIter.next(), new TypeReference<Map<String, Integer>>()
{ {
} }
), ),
(List<EventHolder>) jsonMapper.convertValue( (List<EventHolder>) jsonMapper.convertValue(
resultIter.next(), new TypeReference<List<EventHolder>>() resultIter.next(), new TypeReference<List<EventHolder>>()
{ {
} }
) )
) )
); );

View File

@ -62,7 +62,6 @@ import java.util.Map;
public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<TimeseriesResultValue>, TimeseriesQuery> public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<TimeseriesResultValue>, TimeseriesQuery>
{ {
private static final byte TIMESERIES_QUERY = 0x0; private static final byte TIMESERIES_QUERY = 0x0;
private static final Joiner COMMA_JOIN = Joiner.on(",");
private static final TypeReference<Object> OBJECT_TYPE_REFERENCE = private static final TypeReference<Object> OBJECT_TYPE_REFERENCE =
new TypeReference<Object>() new TypeReference<Object>()
{ {
@ -131,8 +130,20 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest<Result<Timeser
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(DataSourceUtil.getMetricName(query.getDataSource()))
.setUser4("timeseries") .setUser4(query.getType())
.setUser5(COMMA_JOIN.join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());

View File

@ -21,7 +21,6 @@ package io.druid.query.topn;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -65,7 +64,6 @@ import java.util.Map;
public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultValue>, TopNQuery> public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultValue>, TopNQuery>
{ {
private static final byte TOPN_QUERY = 0x1; private static final byte TOPN_QUERY = 0x1;
private static final Joiner COMMA_JOIN = Joiner.on(",");
private static final TypeReference<Result<TopNResultValue>> TYPE_REFERENCE = new TypeReference<Result<TopNResultValue>>() private static final TypeReference<Result<TopNResultValue>> TYPE_REFERENCE = new TypeReference<Result<TopNResultValue>>()
{ {
}; };
@ -147,7 +145,19 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
return new ServiceMetricEvent.Builder() return new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(DataSourceUtil.getMetricName(query.getDataSource()))
.setUser4(String.format("topN/%s/%s", query.getThreshold(), query.getDimensionSpec().getDimension())) .setUser4(String.format("topN/%s/%s", query.getThreshold(), query.getDimensionSpec().getDimension()))
.setUser5(COMMA_JOIN.join(query.getIntervals())) .setUser5(
Lists.transform(
query.getIntervals(),
new Function<Interval, String>()
{
@Override
public String apply(Interval input)
{
return input.toString();
}
}
).toArray(new String[query.getIntervals().size()])
)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size()))
.setUser9(Minutes.minutes(numMinutes).toString()); .setUser9(Minutes.minutes(numMinutes).toString());

View File

@ -20,7 +20,9 @@
package io.druid.server; package io.druid.server;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.EmittingLogger;
import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.emitter.service.ServiceMetricEvent;
@ -41,6 +43,7 @@ import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.proxy.AsyncProxyServlet; import org.eclipse.jetty.proxy.AsyncProxyServlet;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.Interval;
import javax.servlet.AsyncContext; import javax.servlet.AsyncContext;
import javax.servlet.ServletException; import javax.servlet.ServletException;
@ -259,6 +262,12 @@ public class AsyncQueryForwardingServlet extends AsyncProxyServlet
public void onComplete(Result result) public void onComplete(Result result)
{ {
final long requestTime = System.currentTimeMillis() - start; final long requestTime = System.currentTimeMillis() - start;
// toArray() will give compilation error
final String[] intervals = new String[query.getIntervals().size()];
int i = 0;
for (Object interval : query.getIntervals()) {
intervals[i] = interval.toString();
}
try { try {
emitter.emit( emitter.emit(
new ServiceMetricEvent.Builder() new ServiceMetricEvent.Builder()
@ -271,7 +280,7 @@ public class AsyncQueryForwardingServlet extends AsyncProxyServlet
) )
) )
.setUser4(query.getType()) .setUser4(query.getType())
.setUser5(DataSourceUtil.COMMA_JOIN.join(query.getIntervals())) .setUser5(intervals)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser7(req.getRemoteAddr()) .setUser7(req.getRemoteAddr())
.setUser8(query.getId()) .setUser8(query.getId())

View File

@ -23,9 +23,10 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Charsets; import com.google.common.base.Charsets;
import com.google.common.base.Joiner; import com.google.common.base.Function;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams; import com.google.common.io.ByteStreams;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequence;
@ -37,13 +38,13 @@ import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.emitter.service.ServiceMetricEvent;
import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Json;
import io.druid.guice.annotations.Smile; import io.druid.guice.annotations.Smile;
import io.druid.query.DataSourceUtil;
import io.druid.query.Query; import io.druid.query.Query;
import io.druid.query.QueryInterruptedException; import io.druid.query.QueryInterruptedException;
import io.druid.query.QuerySegmentWalker; import io.druid.query.QuerySegmentWalker;
import io.druid.server.initialization.ServerConfig; import io.druid.server.initialization.ServerConfig;
import io.druid.server.log.RequestLogger; import io.druid.server.log.RequestLogger;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.Interval;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -59,6 +60,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.StreamingOutput;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.List;
import java.util.UUID; import java.util.UUID;
/** /**
@ -67,7 +69,6 @@ import java.util.UUID;
public class QueryResource public class QueryResource
{ {
private static final EmittingLogger log = new EmittingLogger(QueryResource.class); private static final EmittingLogger log = new EmittingLogger(QueryResource.class);
private static final Joiner COMMA_JOIN = Joiner.on(",");
public static final String APPLICATION_SMILE = "application/smile"; public static final String APPLICATION_SMILE = "application/smile";
public static final String APPLICATION_JSON = "application/json"; public static final String APPLICATION_JSON = "application/json";
@ -175,9 +176,15 @@ public class QueryResource
try { try {
long requestTime = System.currentTimeMillis() - start; long requestTime = System.currentTimeMillis() - start;
// toArray() will give compilation error
final String[] intervals = new String[query.getIntervals().size()];
int i = 0;
for (Object interval : query.getIntervals()) {
intervals[i] = interval.toString();
}
emitter.emit( emitter.emit(
new ServiceMetricEvent.Builder() new ServiceMetricEvent.Builder()
.setUser2(DataSourceUtil.getMetricName(query.getDataSource())) .setUser2(query.getDataSource().getNames().toArray(new String[query.getDataSource().getNames().size()]))
.setUser3( .setUser3(
jsonMapper.writeValueAsString( jsonMapper.writeValueAsString(
query.getContext() == null query.getContext() == null
@ -186,7 +193,7 @@ public class QueryResource
) )
) )
.setUser4(query.getType()) .setUser4(query.getType())
.setUser5(COMMA_JOIN.join(query.getIntervals())) .setUser5(intervals)
.setUser6(String.valueOf(query.hasFilters())) .setUser6(String.valueOf(query.hasFilters()))
.setUser7(req.getRemoteAddr()) .setUser7(req.getRemoteAddr())
.setUser8(queryId) .setUser8(queryId)