mirror of
https://github.com/apache/druid.git
synced 2025-02-16 23:15:16 +00:00
New property for each metric that tells the StatsDEmitter to convert metric values from range 0-1 to 0-100. This (#3936)
prevents rates and percentages expressed as Doubles (0.xx) from being rounded down to 0.
This commit is contained in:
michaelschiff
Fangjin Yang
parent
ca6053d045
commit
e5fb0e1ff5
@ -29,10 +29,12 @@ All the configuration parameters for the StatsD emitter are under `druid.emitter
|
||||
|
||||
Each metric sent to StatsD must specify a type, one of `[timer, counter, guage]`. StatsD Emitter expects this mapping to
|
||||
be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric.
|
||||
StatsD expects that metric values be integers. Druid emits some metrics with values between the range 0 and 1. To accommodate these metrics they are converted
|
||||
into the range 0 to 100. This conversion can be enabled by setting the optional "convertRange" field true in the JSON mapping file.
|
||||
If the user does not specify their own JSON file, a default mapping is used. All
|
||||
metrics are expected to be mapped. Metrics which are not mapped will log an error.
|
||||
StatsD metric path is organized using the following schema:
|
||||
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>}`
|
||||
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>, "convertRange" : true/false}`
|
||||
e.g.
|
||||
`query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}`
|
||||
|
||||
|
||||
@ -46,7 +46,12 @@ public class DimensionConverter
|
||||
metricMap = readMap(mapper, dimensionMapPath);
|
||||
}
|
||||
|
||||
public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map<String, Object> userDims, ImmutableList.Builder<String> builder)
|
||||
public StatsDMetric addFilteredUserDims(
|
||||
String service,
|
||||
String metric,
|
||||
Map<String, Object> userDims,
|
||||
ImmutableList.Builder<String> builder
|
||||
)
|
||||
{
|
||||
/*
|
||||
Find the metric in the map. If we cant find it try to look it up prefixed by the service name.
|
||||
@ -64,7 +69,7 @@ public class DimensionConverter
|
||||
builder.add(userDims.get(dim).toString());
|
||||
}
|
||||
}
|
||||
return statsDMetric.type;
|
||||
return statsDMetric;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -28,7 +28,6 @@ import com.metamx.emitter.service.ServiceMetricEvent;
|
||||
import com.timgroup.statsd.NonBlockingStatsDClient;
|
||||
import com.timgroup.statsd.StatsDClient;
|
||||
import com.timgroup.statsd.StatsDClientErrorHandler;
|
||||
|
||||
import io.druid.java.util.common.logger.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -47,28 +46,36 @@ public class StatsDEmitter implements Emitter
|
||||
private final StatsDEmitterConfig config;
|
||||
private final DimensionConverter converter;
|
||||
|
||||
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper) {
|
||||
this.config = config;
|
||||
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
|
||||
statsd = new NonBlockingStatsDClient(
|
||||
config.getPrefix(),
|
||||
config.getHostname(),
|
||||
config.getPort(),
|
||||
new StatsDClientErrorHandler()
|
||||
{
|
||||
private int exceptionCount = 0;
|
||||
@Override
|
||||
public void handle(Exception exception)
|
||||
{
|
||||
if (exceptionCount % 1000 == 0) {
|
||||
log.error(exception, "Error sending metric to StatsD.");
|
||||
}
|
||||
exceptionCount += 1;
|
||||
}
|
||||
}
|
||||
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
|
||||
{
|
||||
this(config, mapper,
|
||||
new NonBlockingStatsDClient(
|
||||
config.getPrefix(),
|
||||
config.getHostname(),
|
||||
config.getPort(),
|
||||
new StatsDClientErrorHandler()
|
||||
{
|
||||
private int exceptionCount = 0;
|
||||
|
||||
@Override
|
||||
public void handle(Exception exception)
|
||||
{
|
||||
if (exceptionCount % 1000 == 0) {
|
||||
log.error(exception, "Error sending metric to StatsD.");
|
||||
}
|
||||
exceptionCount += 1;
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper, StatsDClient client)
|
||||
{
|
||||
this.config = config;
|
||||
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
|
||||
this.statsd = client;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {}
|
||||
@ -91,28 +98,29 @@ public class StatsDEmitter implements Emitter
|
||||
nameBuilder.add(service);
|
||||
nameBuilder.add(metric);
|
||||
|
||||
StatsDMetric.Type metricType = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);
|
||||
StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);
|
||||
|
||||
if (metricType != null) {
|
||||
if (statsDMetric != null) {
|
||||
|
||||
String fullName = Joiner.on(config.getSeparator())
|
||||
.join(nameBuilder.build())
|
||||
.replaceAll(DRUID_METRIC_SEPARATOR, config.getSeparator())
|
||||
.replaceAll(STATSD_SEPARATOR, config.getSeparator());
|
||||
|
||||
switch (metricType) {
|
||||
long val = statsDMetric.convertRange ? Math.round(value.doubleValue() * 100) : value.longValue();
|
||||
switch (statsDMetric.type) {
|
||||
case count:
|
||||
statsd.count(fullName, value.longValue());
|
||||
statsd.count(fullName, val);
|
||||
break;
|
||||
case timer:
|
||||
statsd.time(fullName, value.longValue());
|
||||
statsd.time(fullName, val);
|
||||
break;
|
||||
case gauge:
|
||||
statsd.gauge(fullName, value.longValue());
|
||||
statsd.gauge(fullName, val);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
log.error("Metric=[%s] has no StatsD type mapping", metric);
|
||||
log.error("Metric=[%s] has no StatsD type mapping", statsDMetric);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,8 +37,10 @@ import java.util.List;
|
||||
public class StatsDEmitterModule implements DruidModule
|
||||
{
|
||||
private static final String EMITTER_TYPE = "statsd";
|
||||
|
||||
@Override
|
||||
public List<? extends Module> getJacksonModules() {
|
||||
public List<? extends Module> getJacksonModules()
|
||||
{
|
||||
return Collections.EMPTY_LIST;
|
||||
}
|
||||
|
||||
@ -51,7 +53,8 @@ public class StatsDEmitterModule implements DruidModule
|
||||
@Provides
|
||||
@ManageLifecycle
|
||||
@Named(EMITTER_TYPE)
|
||||
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper){
|
||||
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
|
||||
{
|
||||
return new StatsDEmitter(config, mapper);
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,20 +25,27 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.SortedSet;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class StatsDMetric {
|
||||
*/
|
||||
public class StatsDMetric
|
||||
{
|
||||
public final SortedSet<String> dimensions;
|
||||
public final Type type;
|
||||
public final boolean convertRange;
|
||||
|
||||
@JsonCreator
|
||||
public StatsDMetric(
|
||||
@JsonProperty("dimensions") SortedSet<String> dimensions,
|
||||
@JsonProperty("type") Type type)
|
||||
@JsonProperty("type") Type type,
|
||||
@JsonProperty("convertRange") boolean convertRange
|
||||
)
|
||||
{
|
||||
this.dimensions = dimensions;
|
||||
this.type = type;
|
||||
this.convertRange = convertRange;
|
||||
}
|
||||
|
||||
public enum Type {
|
||||
public enum Type
|
||||
{
|
||||
count, gauge, timer
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"query/cache/delta/hits" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/misses" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/evictions" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count", "convertRange" : true },
|
||||
"query/cache/delta/averageBytes" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/timeouts" : { "dimensions" : [], "type" : "count" },
|
||||
"query/cache/delta/errors" : { "dimensions" : [], "type" : "count" },
|
||||
@ -25,7 +25,7 @@
|
||||
"query/cache/total/hits" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/misses" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/evictions" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge", "convertRange" : true },
|
||||
"query/cache/total/averageBytes" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/timeouts" : { "dimensions" : [], "type" : "gauge" },
|
||||
"query/cache/total/errors" : { "dimensions" : [], "type" : "gauge" },
|
||||
@ -65,7 +65,7 @@
|
||||
|
||||
"segment/max" : { "dimensions" : [], "type" : "gauge"},
|
||||
"segment/used" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
|
||||
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
|
||||
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge", "convertRange" : true },
|
||||
|
||||
"jvm/pool/committed" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
|
||||
"jvm/pool/init" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
|
||||
|
||||
@ -35,27 +35,28 @@ public class DimensionConverterTest
|
||||
public void testConvert() throws Exception
|
||||
{
|
||||
DimensionConverter dimensionConverter = new DimensionConverter(new ObjectMapper(), null);
|
||||
ServiceMetricEvent event = new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source")
|
||||
.setDimension("type", "groupBy")
|
||||
.setDimension("interval", "2013/2015")
|
||||
.setDimension("some_random_dim1", "random_dim_value1")
|
||||
.setDimension("some_random_dim2", "random_dim_value2")
|
||||
.setDimension("hasFilters", "no")
|
||||
.setDimension("duration", "P1D")
|
||||
.setDimension("remoteAddress", "194.0.90.2")
|
||||
.setDimension("id", "ID")
|
||||
.setDimension("context", "{context}")
|
||||
.build(new DateTime(), "query/time", 10)
|
||||
.build("broker", "brokerHost1");
|
||||
ServiceMetricEvent event = new ServiceMetricEvent.Builder()
|
||||
.setDimension("dataSource", "data-source")
|
||||
.setDimension("type", "groupBy")
|
||||
.setDimension("interval", "2013/2015")
|
||||
.setDimension("some_random_dim1", "random_dim_value1")
|
||||
.setDimension("some_random_dim2", "random_dim_value2")
|
||||
.setDimension("hasFilters", "no")
|
||||
.setDimension("duration", "P1D")
|
||||
.setDimension("remoteAddress", "194.0.90.2")
|
||||
.setDimension("id", "ID")
|
||||
.setDimension("context", "{context}")
|
||||
.build(new DateTime(), "query/time", 10)
|
||||
.build("broker", "brokerHost1");
|
||||
|
||||
ImmutableList.Builder<String> actual = new ImmutableList.Builder<>();
|
||||
StatsDMetric.Type type = dimensionConverter.addFilteredUserDims(
|
||||
StatsDMetric statsDMetric = dimensionConverter.addFilteredUserDims(
|
||||
event.getService(),
|
||||
event.getMetric(),
|
||||
event.getUserDims(),
|
||||
actual
|
||||
);
|
||||
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, type);
|
||||
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, statsDMetric.type);
|
||||
ImmutableList.Builder<String> expected = new ImmutableList.Builder<>();
|
||||
expected.add("data-source");
|
||||
expected.add("groupBy");
|
||||
|
||||
@ -0,0 +1,111 @@
|
||||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.metamx.emitter.service.ServiceMetricEvent;
|
||||
import com.timgroup.statsd.StatsDClient;
|
||||
import io.druid.emitter.statsd.StatsDEmitter;
|
||||
import io.druid.emitter.statsd.StatsDEmitterConfig;
|
||||
|
||||
import static org.easymock.EasyMock.createMock;
|
||||
import static org.easymock.EasyMock.replay;
|
||||
import static org.easymock.EasyMock.verify;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class StatsDEmitterTest
|
||||
{
|
||||
@Test
|
||||
public void testConvertRange()
|
||||
{
|
||||
StatsDClient client = createMock(StatsDClient.class);
|
||||
StatsDEmitter emitter = new StatsDEmitter(
|
||||
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
|
||||
new ObjectMapper(),
|
||||
client
|
||||
);
|
||||
client.gauge("broker.query.cache.total.hitRate", 54);
|
||||
replay(client);
|
||||
emitter.emit(new ServiceMetricEvent.Builder()
|
||||
.setDimension("dataSource", "data-source")
|
||||
.build(new DateTime(), "query/cache/total/hitRate", 0.54)
|
||||
.build("broker", "brokerHost1")
|
||||
);
|
||||
verify(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoConvertRange()
|
||||
{
|
||||
StatsDClient client = createMock(StatsDClient.class);
|
||||
StatsDEmitter emitter = new StatsDEmitter(
|
||||
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
|
||||
new ObjectMapper(),
|
||||
client
|
||||
);
|
||||
client.time("broker.query.time.data-source.groupBy", 10);
|
||||
replay(client);
|
||||
emitter.emit(new ServiceMetricEvent.Builder()
|
||||
.setDimension("dataSource", "data-source")
|
||||
.setDimension("type", "groupBy")
|
||||
.setDimension("interval", "2013/2015")
|
||||
.setDimension("some_random_dim1", "random_dim_value1")
|
||||
.setDimension("some_random_dim2", "random_dim_value2")
|
||||
.setDimension("hasFilters", "no")
|
||||
.setDimension("duration", "P1D")
|
||||
.setDimension("remoteAddress", "194.0.90.2")
|
||||
.setDimension("id", "ID")
|
||||
.setDimension("context", "{context}")
|
||||
.build(new DateTime(), "query/time", 10)
|
||||
.build("broker", "brokerHost1")
|
||||
);
|
||||
verify(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConfigOptions()
|
||||
{
|
||||
StatsDClient client = createMock(StatsDClient.class);
|
||||
StatsDEmitter emitter = new StatsDEmitter(
|
||||
new StatsDEmitterConfig("localhost", 8888, null, "#", true, null),
|
||||
new ObjectMapper(),
|
||||
client
|
||||
);
|
||||
client.time("brokerHost1#broker#query#time#data-source#groupBy", 10);
|
||||
replay(client);
|
||||
emitter.emit(new ServiceMetricEvent.Builder()
|
||||
.setDimension("dataSource", "data-source")
|
||||
.setDimension("type", "groupBy")
|
||||
.setDimension("interval", "2013/2015")
|
||||
.setDimension("some_random_dim1", "random_dim_value1")
|
||||
.setDimension("some_random_dim2", "random_dim_value2")
|
||||
.setDimension("hasFilters", "no")
|
||||
.setDimension("duration", "P1D")
|
||||
.setDimension("remoteAddress", "194.0.90.2")
|
||||
.setDimension("id", "ID")
|
||||
.setDimension("context", "{context}")
|
||||
.build(new DateTime(), "query/time", 10)
|
||||
.build("broker", "brokerHost1")
|
||||
);
|
||||
verify(client);
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user