diff --git a/docs/content/development/extensions-contrib/statsd.md b/docs/content/development/extensions-contrib/statsd.md index aa89af94d0b..5a150bf9f15 100644 --- a/docs/content/development/extensions-contrib/statsd.md +++ b/docs/content/development/extensions-contrib/statsd.md @@ -44,6 +44,7 @@ All the configuration parameters for the StatsD emitter are under `druid.emitter |`druid.emitter.statsd.includeHost`|Flag to include the hostname as part of the metric name.|no|false| |`druid.emitter.statsd.dimensionMapPath`|JSON file defining the StatsD type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.statsd.blankHolder`|The blank character replacement as statsD does not support path with blank character|no|"-"| +|`druid.emitter.statsd.dogstatsd`|Flag to enable [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/) support. Causes dimensions to be included as tags, not as a part of the metric name. `convertRange` fields will be ignored.|no|false| ### Druid to StatsD Event Converter diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index d8c49abd836..e343795fa27 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -41,9 +41,9 @@ provided - com.timgroup - java-statsd-client - 3.0.1 + com.datadoghq + java-dogstatsd-client + 2.6.1 junit diff --git a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/DimensionConverter.java b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/DimensionConverter.java index 2800e4d3adb..ae1586a3443 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/DimensionConverter.java +++ b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/DimensionConverter.java @@ -22,7 +22,7 @@ package org.apache.druid.emitter.statsd; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.logger.Logger; @@ -49,7 +49,7 @@ public class DimensionConverter String service, String metric, Map userDims, - ImmutableList.Builder builder + ImmutableMap.Builder builder ) { /* @@ -65,7 +65,7 @@ public class DimensionConverter if (statsDMetric != null) { for (String dim : statsDMetric.dimensions) { if (userDims.containsKey(dim)) { - builder.add(userDims.get(dim).toString()); + builder.put(dim, userDims.get(dim).toString()); } } return statsDMetric; diff --git a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitter.java b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitter.java index d2f40b60b47..7794ffee271 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitter.java +++ b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitter.java @@ -22,6 +22,7 @@ package org.apache.druid.emitter.statsd; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; import com.timgroup.statsd.StatsDClientErrorHandler; @@ -33,6 +34,7 @@ import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import java.util.Map; import java.util.regex.Pattern; +import java.util.List; /** */ @@ -43,6 +45,7 @@ public class StatsDEmitter implements Emitter private static final char DRUID_METRIC_SEPARATOR = '/'; private static final Pattern STATSD_SEPARATOR = Pattern.compile("[:|]"); private static final Pattern BLANK = Pattern.compile("\\s+"); + private static final String[] EMPTY_ARRAY = new String[0]; static final StatsDEmitter of(StatsDEmitterConfig config, ObjectMapper mapper) { @@ -50,6 +53,7 @@ public class StatsDEmitter implements Emitter config.getPrefix(), config.getHostname(), config.getPort(), + EMPTY_ARRAY, new StatsDClientErrorHandler() { private int exceptionCount = 0; @@ -93,32 +97,70 @@ public class StatsDEmitter implements Emitter Number value = metricEvent.getValue(); ImmutableList.Builder nameBuilder = new ImmutableList.Builder<>(); - if (config.getIncludeHost()) { - nameBuilder.add(host); - } nameBuilder.add(service); nameBuilder.add(metric); - StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, nameBuilder); + ImmutableMap.Builder dimsBuilder = new ImmutableMap.Builder<>(); + StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, dimsBuilder); if (statsDMetric != null) { + List fullNameList; + String[] tags; + if (config.getDogstatsd()) { + if (config.getIncludeHost()) { + dimsBuilder.put("hostname", host); + } - String fullName = Joiner.on(config.getSeparator()).join(nameBuilder.build()); + fullNameList = nameBuilder.build(); + tags = dimsBuilder.build().entrySet() + .stream() + .map(e -> e.getKey() + ":" + e.getValue()) + .toArray(String[]::new); + } else { + ImmutableList.Builder fullNameBuilder = new ImmutableList.Builder<>(); + if (config.getIncludeHost()) { + fullNameBuilder.add(host); + } + fullNameBuilder.addAll(nameBuilder.build()); + fullNameBuilder.addAll(dimsBuilder.build().values()); + + fullNameList = fullNameBuilder.build(); + tags = EMPTY_ARRAY; + } + + String fullName = Joiner.on(config.getSeparator()).join(fullNameList); fullName = StringUtils.replaceChar(fullName, DRUID_METRIC_SEPARATOR, config.getSeparator()); fullName = STATSD_SEPARATOR.matcher(fullName).replaceAll(config.getSeparator()); fullName = BLANK.matcher(fullName).replaceAll(config.getBlankHolder()); - long val = statsDMetric.convertRange ? Math.round(value.doubleValue() * 100) : value.longValue(); - switch (statsDMetric.type) { - case count: - statsd.count(fullName, val); - break; - case timer: - statsd.time(fullName, val); - break; - case gauge: - statsd.gauge(fullName, val); - break; + if (config.getDogstatsd() && (value instanceof Float || value instanceof Double)) { + switch (statsDMetric.type) { + case count: + statsd.count(fullName, value.doubleValue(), tags); + break; + case timer: + statsd.time(fullName, value.longValue(), tags); + break; + case gauge: + statsd.gauge(fullName, value.doubleValue(), tags); + break; + } + } else { + long val = statsDMetric.convertRange && !config.getDogstatsd() ? + Math.round(value.doubleValue() * 100) : + value.longValue(); + + switch (statsDMetric.type) { + case count: + statsd.count(fullName, val, tags); + break; + case timer: + statsd.time(fullName, val, tags); + break; + case gauge: + statsd.gauge(fullName, val, tags); + break; + } } } else { log.debug("Metric=[%s] has no StatsD type mapping", statsDMetric); diff --git a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitterConfig.java b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitterConfig.java index 8a919ff7bbd..279129b44b8 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitterConfig.java +++ b/extensions-contrib/statsd-emitter/src/main/java/org/apache/druid/emitter/statsd/StatsDEmitterConfig.java @@ -42,6 +42,8 @@ public class StatsDEmitterConfig private final String dimensionMapPath; @JsonProperty private final String blankHolder; + @JsonProperty + private final Boolean dogstatsd; @JsonCreator public StatsDEmitterConfig( @@ -51,7 +53,8 @@ public class StatsDEmitterConfig @JsonProperty("separator") String separator, @JsonProperty("includeHost") Boolean includeHost, @JsonProperty("dimensionMapPath") String dimensionMapPath, - @JsonProperty("blankHolder") String blankHolder + @JsonProperty("blankHolder") String blankHolder, + @JsonProperty("dogstatsd") Boolean dogstatsd ) { this.hostname = Preconditions.checkNotNull(hostname, "StatsD hostname cannot be null."); @@ -61,6 +64,7 @@ public class StatsDEmitterConfig this.includeHost = includeHost != null ? includeHost : false; this.dimensionMapPath = dimensionMapPath; this.blankHolder = blankHolder != null ? blankHolder : "-"; + this.dogstatsd = dogstatsd != null ? dogstatsd : false; } @Override @@ -90,7 +94,10 @@ public class StatsDEmitterConfig if (includeHost != null ? !includeHost.equals(that.includeHost) : that.includeHost != null) { return false; } - return dimensionMapPath != null ? dimensionMapPath.equals(that.dimensionMapPath) : that.dimensionMapPath == null; + if (dimensionMapPath != null ? !dimensionMapPath.equals(that.dimensionMapPath) : that.dimensionMapPath != null) { + return false; + } + return dogstatsd != null ? dogstatsd.equals(that.dogstatsd) : that.dogstatsd == null; } @@ -104,6 +111,7 @@ public class StatsDEmitterConfig result = 31 * result + (includeHost != null ? includeHost.hashCode() : 0); result = 31 * result + (dimensionMapPath != null ? dimensionMapPath.hashCode() : 0); result = 31 * result + (blankHolder != null ? blankHolder.hashCode() : 0); + result = 31 * result + (dogstatsd != null ? dogstatsd.hashCode() : 0); return result; } @@ -148,4 +156,10 @@ public class StatsDEmitterConfig { return blankHolder; } + + @JsonProperty + public Boolean getDogstatsd() + { + return dogstatsd; + } } diff --git a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java index 2ffb06ec26d..8efdb768639 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/DimensionConverterTest.java @@ -20,7 +20,7 @@ package org.apache.druid.emitter.statsd; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.junit.Test; @@ -49,7 +49,7 @@ public class DimensionConverterTest .build(DateTimes.nowUtc(), "query/time", 10) .build("broker", "brokerHost1"); - ImmutableList.Builder actual = new ImmutableList.Builder<>(); + ImmutableMap.Builder actual = new ImmutableMap.Builder<>(); StatsDMetric statsDMetric = dimensionConverter.addFilteredUserDims( event.getService(), event.getMetric(), @@ -57,9 +57,9 @@ public class DimensionConverterTest actual ); assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, statsDMetric.type); - ImmutableList.Builder expected = new ImmutableList.Builder<>(); - expected.add("data-source"); - expected.add("groupBy"); + ImmutableMap.Builder expected = new ImmutableMap.Builder<>(); + expected.put("dataSource", "data-source"); + expected.put("type", "groupBy"); assertEquals("correct Dimensions", expected.build(), actual.build()); } } diff --git a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java index 2346661c77f..1ce3a4ced4b 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/org/apache/druid/emitter/statsd/StatsDEmitterTest.java @@ -38,11 +38,30 @@ public class StatsDEmitterTest { StatsDClient client = createMock(StatsDClient.class); StatsDEmitter emitter = new StatsDEmitter( - new StatsDEmitterConfig("localhost", 8888, null, null, null, null, null), + new StatsDEmitterConfig("localhost", 8888, null, null, null, null, null, null), new ObjectMapper(), client ); - client.gauge("broker.query.cache.total.hitRate", 54); + client.gauge("broker.query.cache.total.hitRate", 54, new String[0]); + replay(client); + emitter.emit(new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .build(DateTimes.nowUtc(), "query/cache/total/hitRate", 0.54) + .build("broker", "brokerHost1") + ); + verify(client); + } + + @Test + public void testConvertRangeWithDogstatsd() + { + StatsDClient client = createMock(StatsDClient.class); + StatsDEmitter emitter = new StatsDEmitter( + new StatsDEmitterConfig("localhost", 8888, null, null, null, null, null, true), + new ObjectMapper(), + client + ); + client.gauge("broker.query.cache.total.hitRate", 0.54, new String[0]); replay(client); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") @@ -57,11 +76,11 @@ public class StatsDEmitterTest { StatsDClient client = createMock(StatsDClient.class); StatsDEmitter emitter = new StatsDEmitter( - new StatsDEmitterConfig("localhost", 8888, null, null, null, null, null), + new StatsDEmitterConfig("localhost", 8888, null, null, null, null, null, null), new ObjectMapper(), client ); - client.time("broker.query.time.data-source.groupBy", 10); + client.time("broker.query.time.data-source.groupBy", 10, new String[0]); replay(client); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") @@ -85,11 +104,40 @@ public class StatsDEmitterTest { StatsDClient client = createMock(StatsDClient.class); StatsDEmitter emitter = new StatsDEmitter( - new StatsDEmitterConfig("localhost", 8888, null, "#", true, null, null), + new StatsDEmitterConfig("localhost", 8888, null, "#", true, null, null, null), new ObjectMapper(), client ); - client.time("brokerHost1#broker#query#time#data-source#groupBy", 10); + client.time("brokerHost1#broker#query#time#data-source#groupBy", 10, new String[0]); + replay(client); + emitter.emit(new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .setDimension("type", "groupBy") + .setDimension("interval", "2013/2015") + .setDimension("some_random_dim1", "random_dim_value1") + .setDimension("some_random_dim2", "random_dim_value2") + .setDimension("hasFilters", "no") + .setDimension("duration", "P1D") + .setDimension("remoteAddress", "194.0.90.2") + .setDimension("id", "ID") + .setDimension("context", "{context}") + .build(DateTimes.nowUtc(), "query/time", 10) + .build("broker", "brokerHost1") + ); + verify(client); + } + + @Test + public void testDogstatsdEnabled() + { + StatsDClient client = createMock(StatsDClient.class); + StatsDEmitter emitter = new StatsDEmitter( + new StatsDEmitterConfig("localhost", 8888, null, "#", true, null, null, true), + new ObjectMapper(), + client + ); + client.time("broker#query#time", 10, + new String[] {"dataSource:data-source", "type:groupBy", "hostname:brokerHost1"}); replay(client); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("dataSource", "data-source") @@ -113,11 +161,11 @@ public class StatsDEmitterTest { StatsDClient client = createMock(StatsDClient.class); StatsDEmitter emitter = new StatsDEmitter( - new StatsDEmitterConfig("localhost", 8888, null, null, true, null, null), + new StatsDEmitterConfig("localhost", 8888, null, null, true, null, null, null), new ObjectMapper(), client ); - client.count("brokerHost1.broker.jvm.gc.count.G1-GC", 1); + client.count("brokerHost1.broker.jvm.gc.count.G1-GC", 1, new String[0]); replay(client); emitter.emit(new ServiceMetricEvent.Builder() .setDimension("gcName", "G1 GC")