Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

prometheus-emitter: add extraLabels parameter (#14728) 

* prometheus-emitter: add extraLabels parameter

* prometheus-emitter: update readme to include the extraLabels parameter

* prometheus-emitter: remove nullable and surface label name issues

* remove import to make linter happy
This commit is contained in:
John Gerassimou 2023-08-29 15:02:22 -04:00 committed by GitHub
parent 004cd012e1
commit d201ea0ece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 280 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/development/extensions-contrib/prometheus.md
View File

@ -46,6 +46,7 @@ All the configuration parameters for the Prometheus emitter are under `druid.emi
| `druid.emitter.prometheus.addServiceAsLabel` | Flag to include the druid service name (e.g. `druid/broker`, `druid/coordinator`, etc.) as a prometheus label. | no | false |
| `druid.emitter.prometheus.pushGatewayAddress` | Pushgateway address. Required if using `pushgateway` strategy. | no | none |
| `druid.emitter.prometheus.flushPeriod` | Emit metrics to Pushgateway every `flushPeriod` seconds. Required if `pushgateway` strategy is used. | no | 15 |
| `druid.emitter.prometheus.extraLabels` | JSON key-value pairs for additional labels on all metrics. Keys (label names) must match the regex `[a-zA-Z_:][a-zA-Z0-9_:]*`. Example: `{"cluster_name": "druid_cluster1", "env": "staging"}`. | no | none |
### Ports for colocated Druid processes

9
extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
View File

@ -62,10 +62,15 @@ public class Metrics
}
}
public Metrics(String namespace, String path, boolean isAddHostAsLabel, boolean isAddServiceAsLabel)
public Metrics(String namespace, String path, boolean isAddHostAsLabel, boolean isAddServiceAsLabel, Map<String, String> extraLabels)
{
Map<String, DimensionsAndCollector> registeredMetrics = new HashMap<>();
Map<String, Metric> metrics = readConfig(path);
if (extraLabels == null) {
extraLabels = Collections.emptyMap(); // Avoid null checks later
}
for (Map.Entry<String, Metric> entry : metrics.entrySet()) {
String name = entry.getKey();
Metric metric = entry.getValue();
@ -79,6 +84,8 @@ public class Metrics
metric.dimensions.add(TAG_SERVICE);
}
metric.dimensions.addAll(extraLabels.keySet());
String[] dimensions = metric.dimensions.toArray(new String[0]);
String formattedName = PATTERN.matcher(StringUtils.toLowerCase(name)).replaceAll("_");
SimpleCollector collector = null;

7
extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
View File

@ -69,7 +69,7 @@ public class PrometheusEmitter implements Emitter
{
this.config = config;
this.strategy = config.getStrategy();
metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath(), config.isAddHostAsLabel(), config.isAddServiceAsLabel());
metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath(), config.isAddHostAsLabel(), config.isAddServiceAsLabel(), config.getExtraLabels());
}
@ -136,6 +136,9 @@ public class PrometheusEmitter implements Emitter
if (metric != null) {
String[] labelValues = new String[metric.getDimensions().length];
String[] labelNames = metric.getDimensions();
Map<String, String> extraLabels = config.getExtraLabels();
for (int i = 0; i < labelValues.length; i++) {
String labelName = labelNames[i];
//labelName is controlled by the user. Instead of potential NPE on invalid labelName we use "unknown" as the dimension value
@ -148,6 +151,8 @@ public class PrometheusEmitter implements Emitter
labelValues[i] = host;
} else if (config.isAddServiceAsLabel() && TAG_SERVICE.equals(labelName)) {
labelValues[i] = service;
} else if (extraLabels.containsKey(labelName)) {
labelValues[i] = config.getExtraLabels().get(labelName);
} else {
labelValues[i] = "unknown";
}

30
extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
View File

@ -22,8 +22,12 @@ package org.apache.druid.emitter.prometheus;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.error.DruidException;
import org.apache.druid.java.util.common.StringUtils;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
@ -63,6 +67,9 @@ public class PrometheusEmitterConfig
@JsonProperty
private final boolean addServiceAsLabel;
@JsonProperty
private final Map<String, String> extraLabels;
@JsonCreator
public PrometheusEmitterConfig(
@JsonProperty("strategy") @Nullable Strategy strategy,
@ -72,7 +79,8 @@ public class PrometheusEmitterConfig
@JsonProperty("pushGatewayAddress") @Nullable String pushGatewayAddress,
@JsonProperty("addHostAsLabel") boolean addHostAsLabel,
@JsonProperty("addServiceAsLabel") boolean addServiceAsLabel,
@JsonProperty("flushPeriod") Integer flushPeriod
@JsonProperty("flushPeriod") Integer flushPeriod,
@JsonProperty("extraLabels") @Nullable Map<String, String> extraLabels
)
{
this.strategy = strategy != null ? strategy : Strategy.exporter;
@ -94,6 +102,21 @@ public class PrometheusEmitterConfig
this.flushPeriod = flushPeriod;
this.addHostAsLabel = addHostAsLabel;
this.addServiceAsLabel = addServiceAsLabel;
this.extraLabels = extraLabels != null ? extraLabels : Collections.emptyMap();
// Validate label names early to prevent Prometheus exceptions later.
for (String key : this.extraLabels.keySet()) {
if (!PATTERN.matcher(key).matches()) {
throw DruidException.forPersona(DruidException.Persona.OPERATOR)
.ofCategory(DruidException.Category.INVALID_INPUT)
.build(
StringUtils.format(
"Invalid metric label name [%s]. Label names must conform to the pattern [%s].",
key,
PATTERN.pattern()
)
);
}
}
}
public String getNamespace()
@ -137,6 +160,11 @@ public class PrometheusEmitterConfig
return addServiceAsLabel;
}
public Map<String, String> getExtraLabels()
{
return extraLabels;
}
public enum Strategy
{
exporter, pushgateway

49
extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
View File

@ -23,12 +23,15 @@ import io.prometheus.client.Histogram;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
public class MetricsTest
{
@Test
public void testMetricsConfiguration()
{
Metrics metrics = new Metrics("test", null, true, true);
Metrics metrics = new Metrics("test", null, true, true, null);
DimensionsAndCollector dimensionsAndCollector = metrics.getByName("query/time", "historical");
Assert.assertNotNull(dimensionsAndCollector);
String[] dimensions = dimensionsAndCollector.getDimensions();
@ -46,4 +49,48 @@ public class MetricsTest
Assert.assertEquals("host_name", dims[1]);
Assert.assertEquals("server", dims[2]);
}
@Test
public void testMetricsConfigurationWithExtraLabels()
{
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("extra_label", "value");
Metrics metrics = new Metrics("test_2", null, true, true, extraLabels);
DimensionsAndCollector dimensionsAndCollector = metrics.getByName("query/time", "historical");
Assert.assertNotNull(dimensionsAndCollector);
String[] dimensions = dimensionsAndCollector.getDimensions();
Assert.assertEquals("dataSource", dimensions[0]);
Assert.assertEquals("druid_service", dimensions[1]);
Assert.assertEquals("extra_label", dimensions[2]);
Assert.assertEquals("host_name", dimensions[3]);
Assert.assertEquals("type", dimensions[4]);
Assert.assertEquals(1000.0, dimensionsAndCollector.getConversionFactor(), 0.0);
Assert.assertTrue(dimensionsAndCollector.getCollector() instanceof Histogram);
DimensionsAndCollector d = metrics.getByName("segment/loadQueue/count", "historical");
Assert.assertNotNull(d);
String[] dims = d.getDimensions();
Assert.assertEquals("druid_service", dims[0]);
Assert.assertEquals("extra_label", dims[1]);
Assert.assertEquals("host_name", dims[2]);
Assert.assertEquals("server", dims[3]);
}
@Test
public void testMetricsConfigurationWithBadExtraLabels()
{
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("extra label", "value");
// Expect an exception thrown by Prometheus code due to invalid metric label
Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> {
new Metrics("test_3", null, true, true, extraLabels);
});
String expectedMessage = "Invalid metric label name: extra label";
String actualMessage = exception.getMessage();
Assert.assertTrue(actualMessage.contains(expectedMessage));
}
}

51
extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfigTest.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.emitter.prometheus;
import io.prometheus.client.CollectorRegistry;
import org.apache.druid.error.DruidException;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
public class PrometheusEmitterConfigTest
{
@Test
public void testEmitterConfigWithBadExtraLabels()
{
CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("label Name", "label Value");
// Expect an exception thrown by our own PrometheusEmitterConfig due to invalid label key
Exception exception = Assert.assertThrows(DruidException.class, () -> {
new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, extraLabels);
});
String expectedMessage = "Invalid metric label name [label Name]. Label names must conform to the pattern [[a-zA-Z_:][a-zA-Z0-9_:]*]";
String actualMessage = exception.getMessage();
Assert.assertTrue(actualMessage.contains(expectedMessage));
}
}

149
extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
View File

@ -29,19 +29,20 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.mock;
public class PrometheusEmitterTest
{
@Test
public void testEmitterWithServiceLabel()
{
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60);
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, null);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
@ -62,7 +63,7 @@ public class PrometheusEmitterTest
public void testEmitterWithServiceAndHostLabel()
{
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, true, true, 60);
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, true, true, 60, null);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
@ -79,11 +80,129 @@ public class PrometheusEmitterTest
Assert.assertEquals(10, count.intValue());
}
@Test
public void testEmitterWithExtraLabels()
{
CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName", "labelValue");
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, false, 60, extraLabels);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"labelName", "server"},
new String[]{"labelValue", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}
@Test
public void testEmitterWithServiceLabelAndExtraLabel()
{
CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName", "labelValue");
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, extraLabels);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"druid_service", "labelName", "server"},
new String[]{"historical", "labelValue", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}
@Test
public void testEmitterWithEmptyExtraLabels()
{
CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, extraLabels);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"druid_service", "server"},
new String[]{"historical", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}
@Test
public void testEmitterWithMultipleExtraLabels()
{
CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName1", "labelValue1");
extraLabels.put("labelName2", "labelValue2");
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, extraLabels);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"druid_service", "labelName1", "labelName2", "server"},
new String[]{"historical", "labelValue1", "labelValue2", "druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}
@Test
public void testEmitterWithLabelCollision()
{
CollectorRegistry.defaultRegistry.clear();
// ExtraLabels contains a label that collides with a service label
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("server", "collisionLabelValue");
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null, 0, null, false, true, 60, extraLabels);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server", "druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"druid_service", "server"},
new String[]{"historical", "druid_data01_vpc_region"}
);
// Check that the extraLabel did not override the service label
Assert.assertEquals(10, count.intValue());
// Check that the extraLabel is not present in the CollectorRegistry
Assert.assertNull(CollectorRegistry.defaultRegistry.getSampleValue(
"druid_segment_loadqueue_count",
new String[]{"druid_service", "server"},
new String[]{"historical", "collisionLabelValue"}
));
}
@Test
public void testEmitterMetric()
{
CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace", null, 0, "pushgateway", true, true, 60);
PrometheusEmitterConfig config = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace", null, 0, "pushgateway", true, true, 60, null);
PrometheusEmitterModule prometheusEmitterModule = new PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
@ -104,12 +223,12 @@ public class PrometheusEmitterTest
@Test
public void testEmitterStart()
{
PrometheusEmitterConfig exportEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, "namespace1", null, 0, null, true, true, 60);
PrometheusEmitterConfig exportEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, "namespace1", null, 0, null, true, true, 60, null);
PrometheusEmitter exportEmitter = new PrometheusEmitter(exportEmitterConfig);
exportEmitter.start();
Assert.assertNotNull(exportEmitter.getServer());
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace2", null, 0, "pushgateway", true, true, 60);
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace2", null, 0, "pushgateway", true, true, 60, null);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
@ -118,7 +237,7 @@ public class PrometheusEmitterTest
@Test
public void testEmitterPush() throws IOException
{
PrometheusEmitterConfig emitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace3", null, 0, "pushgateway", true, true, 60);
PrometheusEmitterConfig emitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace3", null, 0, "pushgateway", true, true, 60, null);
PushGateway mockPushGateway = mock(PushGateway.class);
mockPushGateway.push(anyObject(Collector.class), anyString(), anyObject(ImmutableMap.class));
@ -146,7 +265,8 @@ public class PrometheusEmitterTest
null,
true,
true,
60
60,
null
);
Assert.assertThrows(
@ -160,7 +280,8 @@ public class PrometheusEmitterTest
null,
true,
true,
50
50,
null
)
);
}
@ -168,7 +289,7 @@ public class PrometheusEmitterTest
@Test
public void testEmitterStartWithHttpUrl()
{
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, "http://pushgateway", true, true, 60);
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace4", null, 0, "http://pushgateway", true, true, 60, null);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
@ -177,7 +298,7 @@ public class PrometheusEmitterTest
@Test
public void testEmitterStartWithHttpsUrl()
{
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace5", null, 0, "https://pushgateway", true, true, 60);
PrometheusEmitterConfig pushEmitterConfig = new PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway, "namespace5", null, 0, "https://pushgateway", true, true, 60, null);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
@ -197,7 +318,8 @@ public class PrometheusEmitterTest
"https://pushgateway",
true,
true,
60
60,
null
)
);
@ -210,7 +332,8 @@ public class PrometheusEmitterTest
"https://pushgateway",
true,
true,
60
60,
null
);
}
}