Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix graphite whitelist converter to handle array dimensions (#5269) 

* Fix graphite whitelist converter to handle array dimensions

* Fix ambari whitelist converter to handle array dimensions
This commit is contained in:
Kevin Conaway 2018-01-29 11:16:46 -05:00 committed by Nishant Bangarwa
parent 2a892709e8
commit a5ba31c230
6 changed files with 120 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
View File

@ -42,6 +42,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.SortedMap;
@ -138,15 +139,27 @@ public class WhiteListBasedDruidToTimelineEventConverter implements DruidToTimel
if (prefixKey == null) {
return null;
}
ImmutableList.Builder<String> outputList = new ImmutableList.Builder();
ImmutableList.Builder<String> outputList = new ImmutableList.Builder<>();
List<String> dimensions = whiteListDimsMapper.get(prefixKey);
if (dimensions == null) {
return Collections.emptyList();
}
for (String dimKey : dimensions) {
String dimValue = (String) event.getUserDims().get(dimKey);
if (dimValue != null) {
outputList.add(AmbariMetricsEmitter.sanitize(dimValue));
Object rawValue = event.getUserDims().get(dimKey);
String value = null;
if (rawValue instanceof String) {
value = (String) rawValue;
} else if (rawValue instanceof Collection) {
Collection values = (Collection) rawValue;
if (!values.isEmpty()) {
value = (String) values.iterator().next();
}
}
if (value != null) {
outputList.add(AmbariMetricsEmitter.sanitize(value));
}
}
return outputList.build();

37
extensions-contrib/ambari-metrics-emitter/src/test/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java
View File

@ -25,6 +25,7 @@ import io.druid.jackson.DefaultObjectMapper;
import io.druid.java.util.common.DateTimes;
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.easymock.EasyMock;
import org.joda.time.DateTime;
@ -33,6 +34,11 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@RunWith(JUnitParamsRunner.class)
public class WhiteListBasedDruidToTimelineEventConverterTest
@ -106,6 +112,37 @@ public class WhiteListBasedDruidToTimelineEventConverterTest
Assert.assertEquals(expectedPath, path);
}
@Test
public void testWhiteListedStringArrayDimension() throws IOException
{
File mapFile = File.createTempFile("testing-" + System.nanoTime(), ".json");
mapFile.deleteOnExit();
try (OutputStream outputStream = new FileOutputStream(mapFile)) {
IOUtils.copyLarge(
getClass().getResourceAsStream("/testWhiteListedStringArrayDimension.json"),
outputStream
);
}
WhiteListBasedDruidToTimelineEventConverter converter = new WhiteListBasedDruidToTimelineEventConverter(
prefix,
"druid",
mapFile.getAbsolutePath(),
new DefaultObjectMapper()
);
ServiceMetricEvent event = new ServiceMetricEvent.Builder()
.setDimension("gcName", new String[] {"g1"})
.build(createdTime, "jvm/gc/cpu", 10)
.build(serviceName, hostname);
TimelineMetric metric = converter.druidEventToTimelineMetric(event);
Assert.assertNotNull(metric);
Assert.assertEquals(defaultNamespace + ".g1.jvm/gc/cpu", metric.getMetricName());
}
private Object[] parametersForTestGetName()
{
return new Object[]{

3
extensions-contrib/ambari-metrics-emitter/src/test/resources/testWhiteListedStringArrayDimension.json Normal file
View File

@ -0,0 +1,3 @@
{
"jvm/gc": ["gcName"]
}

28
extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java
View File

@ -41,6 +41,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@ -169,15 +170,26 @@ public class WhiteListBasedConverter implements DruidToGraphiteEventConverter
if (prefixKey == null) {
return null;
}
ImmutableList.Builder<String> outputList = new ImmutableList.Builder();
ImmutableList.Builder<String> outputList = new ImmutableList.Builder<>();
Set<String> dimensions = whiteListDimsMapper.get(prefixKey);
if (dimensions == null) {
return Collections.emptyList();
}
for (String dimKey : dimensions) {
String dimValue = (String) event.getUserDims().get(dimKey);
if (dimValue != null) {
outputList.add(GraphiteEmitter.sanitize(dimValue));
Object rawValue = event.getUserDims().get(dimKey);
String value = null;
if (rawValue instanceof String) {
value = (String) rawValue;
} else if (rawValue instanceof Collection) {
Collection values = (Collection) rawValue;
if (!values.isEmpty()) {
value = (String) values.iterator().next();
}
}
if (value != null) {
outputList.add(GraphiteEmitter.sanitize(value));
}
}
return outputList.build();
@ -210,15 +222,17 @@ public class WhiteListBasedConverter implements DruidToGraphiteEventConverter
if (!this.isIgnoreHostname()) {
metricPathBuilder.add(GraphiteEmitter.sanitize(serviceMetricEvent.getHost()));
}
metricPathBuilder.addAll(this.getOrderedDimValues(serviceMetricEvent));
List<String> dimValues = getOrderedDimValues(serviceMetricEvent);
if (dimValues != null) {
metricPathBuilder.addAll(dimValues);
}
metricPathBuilder.add(GraphiteEmitter.sanitize(serviceMetricEvent.getMetric(), this.replaceSlashWithDot()));
final GraphiteEvent graphiteEvent = new GraphiteEvent(
return new GraphiteEvent(
Joiner.on(".").join(metricPathBuilder.build()),
String.valueOf(serviceMetricEvent.getValue()),
TimeUnit.MILLISECONDS.toSeconds(serviceMetricEvent.getCreatedTime().getMillis())
);
return graphiteEvent;
}
@Override

39
extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java
View File

@ -25,6 +25,7 @@ import io.druid.jackson.DefaultObjectMapper;
import io.druid.java.util.common.DateTimes;
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
import org.apache.commons.io.IOUtils;
import org.easymock.EasyMock;
import org.joda.time.DateTime;
import org.junit.Assert;
@ -32,6 +33,11 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@RunWith(JUnitParamsRunner.class)
public class WhiteListBasedConverterTest
@ -106,6 +112,39 @@ public class WhiteListBasedConverterTest
Assert.assertEquals(expectedPath, path);
}
@Test
public void testWhiteListedStringArrayDimension() throws IOException
{
File mapFile = File.createTempFile("testing-" + System.nanoTime(), ".json");
mapFile.deleteOnExit();
try (OutputStream outputStream = new FileOutputStream(mapFile)) {
IOUtils.copyLarge(
getClass().getResourceAsStream("/testWhiteListedStringArrayDimension.json"),
outputStream
);
}
WhiteListBasedConverter converter = new WhiteListBasedConverter(
prefix,
false,
false,
false,
mapFile.getAbsolutePath(),
new DefaultObjectMapper()
);
ServiceMetricEvent event = new ServiceMetricEvent.Builder()
.setDimension("gcName", new String[] {"g1"})
.build(createdTime, "jvm/gc/cpu", 10)
.build(serviceName, hostname);
GraphiteEvent graphiteEvent = converter.druidEventToGraphite(event);
Assert.assertNotNull(graphiteEvent);
Assert.assertEquals(defaultNamespace + ".g1.jvm/gc/cpu", graphiteEvent.getEventPath());
}
private Object[] parametersForTestGetPath()
{
return new Object[]{

3
extensions-contrib/graphite-emitter/src/test/resources/testWhiteListedStringArrayDimension.json Normal file
View File

@ -0,0 +1,3 @@
{
"jvm/gc": ["gcName"]
}