Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New property for each metric that tells the StatsDEmitter to convert metric values from range 0-1 to 0-100. This (#3936) 

prevents rates and percentages expressed as Doubles (0.xx) from being rounded down to 0.
This commit is contained in:
michaelschiff 2017-02-16 13:55:56 -08:00 committed by Fangjin Yang
parent ca6053d045
commit e5fb0e1ff5
8 changed files with 190 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/content/development/extensions-contrib/statsd.md
View File

@ -29,10 +29,12 @@ All the configuration parameters for the StatsD emitter are under `druid.emitter
Each metric sent to StatsD must specify a type, one of `[timer, counter, guage]`. StatsD Emitter expects this mapping to
be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric.
StatsD expects that metric values be integers. Druid emits some metrics with values between the range 0 and 1. To accommodate these metrics they are converted
into the range 0 to 100. This conversion can be enabled by setting the optional "convertRange" field true in the JSON mapping file.
If the user does not specify their own JSON file, a default mapping is used. All
metrics are expected to be mapped. Metrics which are not mapped will log an error.
StatsD metric path is organized using the following schema:
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>}`
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>, "convertRange" : true/false}`
e.g.
`query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}`

9
extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/DimensionConverter.java
View File

@ -46,7 +46,12 @@ public class DimensionConverter
metricMap = readMap(mapper, dimensionMapPath);
}
public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map<String, Object> userDims, ImmutableList.Builder<String> builder)
public StatsDMetric addFilteredUserDims(
String service,
String metric,
Map<String, Object> userDims,
ImmutableList.Builder<String> builder
)
{
/*
Find the metric in the map. If we cant find it try to look it up prefixed by the service name.
@ -64,7 +69,7 @@ public class DimensionConverter
builder.add(userDims.get(dim).toString());
}
}
return statsDMetric.type;
return statsDMetric;
} else {
return null;
}

62
extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java
View File

@ -28,7 +28,6 @@ import com.metamx.emitter.service.ServiceMetricEvent;
import com.timgroup.statsd.NonBlockingStatsDClient;
import com.timgroup.statsd.StatsDClient;
import com.timgroup.statsd.StatsDClientErrorHandler;
import io.druid.java.util.common.logger.Logger;
import java.io.IOException;
@ -47,28 +46,36 @@ public class StatsDEmitter implements Emitter
private final StatsDEmitterConfig config;
private final DimensionConverter converter;
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper) {
this.config = config;
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
statsd = new NonBlockingStatsDClient(
config.getPrefix(),
config.getHostname(),
config.getPort(),
new StatsDClientErrorHandler()
{
private int exceptionCount = 0;
@Override
public void handle(Exception exception)
{
if (exceptionCount % 1000 == 0) {
log.error(exception, "Error sending metric to StatsD.");
}
exceptionCount += 1;
}
}
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
{
this(config, mapper,
new NonBlockingStatsDClient(
config.getPrefix(),
config.getHostname(),
config.getPort(),
new StatsDClientErrorHandler()
{
private int exceptionCount = 0;
@Override
public void handle(Exception exception)
{
if (exceptionCount % 1000 == 0) {
log.error(exception, "Error sending metric to StatsD.");
}
exceptionCount += 1;
}
}
)
);
}
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper, StatsDClient client)
{
this.config = config;
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
this.statsd = client;
}
@Override
public void start() {}
@ -91,28 +98,29 @@ public class StatsDEmitter implements Emitter
nameBuilder.add(service);
nameBuilder.add(metric);
StatsDMetric.Type metricType = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);
StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);
if (metricType != null) {
if (statsDMetric != null) {
String fullName = Joiner.on(config.getSeparator())
.join(nameBuilder.build())
.replaceAll(DRUID_METRIC_SEPARATOR, config.getSeparator())
.replaceAll(STATSD_SEPARATOR, config.getSeparator());
switch (metricType) {
long val = statsDMetric.convertRange ? Math.round(value.doubleValue() * 100) : value.longValue();
switch (statsDMetric.type) {
case count:
statsd.count(fullName, value.longValue());
statsd.count(fullName, val);
break;
case timer:
statsd.time(fullName, value.longValue());
statsd.time(fullName, val);
break;
case gauge:
statsd.gauge(fullName, value.longValue());
statsd.gauge(fullName, val);
break;
}
} else {
log.error("Metric=[%s] has no StatsD type mapping", metric);
log.error("Metric=[%s] has no StatsD type mapping", statsDMetric);
}
}
}

7
extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java
View File

@ -37,8 +37,10 @@ import java.util.List;
public class StatsDEmitterModule implements DruidModule
{
private static final String EMITTER_TYPE = "statsd";
@Override
public List<? extends Module> getJacksonModules() {
public List<? extends Module> getJacksonModules()
{
return Collections.EMPTY_LIST;
}
@ -51,7 +53,8 @@ public class StatsDEmitterModule implements DruidModule
@Provides
@ManageLifecycle
@Named(EMITTER_TYPE)
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper){
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
{
return new StatsDEmitter(config, mapper);
}
}

15
extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDMetric.java
View File

@ -25,20 +25,27 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.SortedSet;
/**
*/
public class StatsDMetric {
*/
public class StatsDMetric
{
public final SortedSet<String> dimensions;
public final Type type;
public final boolean convertRange;
@JsonCreator
public StatsDMetric(
@JsonProperty("dimensions") SortedSet<String> dimensions,
@JsonProperty("type") Type type)
@JsonProperty("type") Type type,
@JsonProperty("convertRange") boolean convertRange
)
{
this.dimensions = dimensions;
this.type = type;
this.convertRange = convertRange;
}
public enum Type {
public enum Type
{
count, gauge, timer
}
}

6
extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
View File

@ -15,7 +15,7 @@
"query/cache/delta/hits" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/misses" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/evictions" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count", "convertRange" : true },
"query/cache/delta/averageBytes" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/timeouts" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/errors" : { "dimensions" : [], "type" : "count" },
@ -25,7 +25,7 @@
"query/cache/total/hits" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/misses" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/evictions" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge", "convertRange" : true },
"query/cache/total/averageBytes" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/timeouts" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/errors" : { "dimensions" : [], "type" : "gauge" },
@ -65,7 +65,7 @@
"segment/max" : { "dimensions" : [], "type" : "gauge"},
"segment/used" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge", "convertRange" : true },
"jvm/pool/committed" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
"jvm/pool/init" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },

29
extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java
View File

@ -35,27 +35,28 @@ public class DimensionConverterTest
public void testConvert() throws Exception
{
DimensionConverter dimensionConverter = new DimensionConverter(new ObjectMapper(), null);
ServiceMetricEvent event = new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1");
ServiceMetricEvent event = new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1");
ImmutableList.Builder<String> actual = new ImmutableList.Builder<>();
StatsDMetric.Type type = dimensionConverter.addFilteredUserDims(
StatsDMetric statsDMetric = dimensionConverter.addFilteredUserDims(
event.getService(),
event.getMetric(),
event.getUserDims(),
actual
);
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, type);
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, statsDMetric.type);
ImmutableList.Builder<String> expected = new ImmutableList.Builder<>();
expected.add("data-source");
expected.add("groupBy");

111
extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import com.fasterxml.jackson.databind.ObjectMapper;
import com.metamx.emitter.service.ServiceMetricEvent;
import com.timgroup.statsd.StatsDClient;
import io.druid.emitter.statsd.StatsDEmitter;
import io.druid.emitter.statsd.StatsDEmitterConfig;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import org.joda.time.DateTime;
import org.junit.Test;
/**
*/
public class StatsDEmitterTest
{
@Test
public void testConvertRange()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
new ObjectMapper(),
client
);
client.gauge("broker.query.cache.total.hitRate", 54);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.build(new DateTime(), "query/cache/total/hitRate", 0.54)
.build("broker", "brokerHost1")
);
verify(client);
}
@Test
public void testNoConvertRange()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
new ObjectMapper(),
client
);
client.time("broker.query.time.data-source.groupBy", 10);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1")
);
verify(client);
}
@Test
public void testConfigOptions()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, "#", true, null),
new ObjectMapper(),
client
);
client.time("brokerHost1#broker#query#time#data-source#groupBy", 10);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1")
);
verify(client);
}
}