diff --git a/.idea/misc.xml b/.idea/misc.xml index 61e7e5b21e7..bf2061d7392 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -87,4 +87,4 @@ - \ No newline at end of file + diff --git a/distribution/pom.xml b/distribution/pom.xml index 76466cf9c87..215cd8d95b1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -373,6 +373,8 @@ -c org.apache.druid.extensions.contrib:ambari-metrics-emitter -c + org.apache.druid.extensions.contrib:dropwizard-emitter + -c org.apache.druid.extensions.contrib:druid-azure-extensions -c org.apache.druid.extensions.contrib:druid-cassandra-storage diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md new file mode 100644 index 00000000000..283954b304e --- /dev/null +++ b/docs/design/extensions-contrib/dropwizard.md @@ -0,0 +1,630 @@ +--- +layout: doc_page +title: "Dropwizard metrics emitter" +--- + + + +# Dropwizard Emitter + +To use this extension, make sure to [include](../../development/extensions.md#loading-extensions) `dropwizard-emitter` extension. + +## Introduction + +This extension integrates [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#) metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. +It accumulates druid metrics as dropwizard metrics, and emits them to various sinks via dropwizard supported reporters. +Currently supported dropwizard metrics types counter, gauge, meter, timer and histogram. +These metrics can be emitted using either Console or JMX reporter. + +## Configuration + +All the configuration parameters for Dropwizard emitter are under `druid.emitter.dropwizard`. + +|property|description|required?|default| +|--------|-----------|---------|-------| +|`druid.emitter.dropwizard.reporters`|List of dropwizard reporters to be used. Here is a list of [Supported Reporters](#supported-dropwizard-reporters)|yes|none| +|`druid.emitter.dropwizard.prefix`|Optional prefix to be used for metrics name|no|none| +|`druid.emitter.dropwizard.includeHost`|Flag to include the host and port as part of the metric name.|no|yes| +|`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| +|`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| +|`druid.emitter.dropwizard.maxMetricsRegistrySize`| Maximum size of metrics registry to be cached at any time. |no| 100 Mb| + + +### Druid to Dropwizard Event Conversion + +Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram]`. Dropwizard Emitter expects this mapping to +be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. +If the user does not specify their own JSON file, a [default mapping](#default-metrics-mapping) is used. +All metrics are expected to be mapped. Metrics which are not mapped will be ignored. +Dropwizard metric path is organized using the following schema: + +` : { "dimensions" : , "type" : , "timeUnit" : }` + +e.g. +```json +"query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer", "timeUnit": "MILLISECONDS"}, +"segment/scan/pending" : { "dimensions" : [], "type" : "gauge"} +``` + +For most use-cases, the default mapping is sufficient. + +### Supported Dropwizard reporters + +#### JMX Reporter +Used to report druid metrics via JMX. +``` + +druid.emitter.dropwizard.reporters={"type":"jmx"} + +``` + +#### Console Reporter +Used to print Druid Metrics to console logs. + +``` + +druid.emitter.dropwizard.reporters={"type":"console","emitIntervalInSecs":30}"} + +``` + +### Default Metrics Mapping +Latest default metrics mapping can be found [here] (https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) +```json +{ + "query/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/time": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/ttfb": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/backpressure": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/intervalChunk/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/segment/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/wait/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/scan/pending": { + "dimensions": [], + "type": "gauge" + }, + "query/segmentAndCache/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/cpu/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "query/cache/delta/numEntries": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/sizeBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hits": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/misses": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/evictions": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hitRate": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/averageBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/timeouts": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/errors": { + "dimensions": [], + "type": "counter" + }, + "query/cache/total/numEntries": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/sizeBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hits": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/misses": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/evictions": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hitRate": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/averageBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/timeouts": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/errors": { + "dimensions": [], + "type": "gauge" + }, + "ingest/events/thrownAway": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/unparseable": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/duplicate": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/processed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/rows/output": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/counter": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/persist/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/persist/backPressure": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "ingest/persist/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/handoff/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/merge/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/merge/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "task/run/time": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/added/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/moved/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/nuked/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/assigned/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/moved/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/dropped/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/deleted/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/unneeded/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/raw": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalization": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalized": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/loadQueue/size": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/failed": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/dropQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/size": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "segment/overShadowed/counter": { + "dimensions": [], + "type": "gauge" + }, + "segment/max": { + "dimensions": [], + "type": "gauge" + }, + "segment/used": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "segment/usedPercent": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "jvm/pool/committed": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/init": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/max": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/used": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/counter": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/used": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/capacity": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/mem/init": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/max": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/used": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/committed": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/gc/counter": { + "dimensions": [ + "gcName" + ], + "type": "counter" + }, + "jvm/gc/cpu": { + "dimensions": [ + "gcName" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/events/buffered": { + "dimensions": [ + "serviceName, bufferCapacity" + ], + "type": "gauge" + }, + "sys/swap/free": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageIn": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageOut": { + "dimensions": [], + "type": "gauge" + }, + "sys/disk/write/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/write/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/net/write/size": { + "dimensions": [], + "type": "counter" + }, + "sys/net/read/size": { + "dimensions": [], + "type": "counter" + }, + "sys/fs/used": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/fs/max": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/mem/used": { + "dimensions": [], + "type": "gauge" + }, + "sys/mem/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/storage/used": { + "dimensions": [ + "fsDirName" + ], + "type": "gauge" + }, + "sys/cpu": { + "dimensions": [ + "cpuName", + "cpuTime" + ], + "type": "gauge" + }, + "coordinator-segment/counter": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "historical-segment/counter": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + } +} +``` \ No newline at end of file diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml new file mode 100644 index 00000000000..def2c20c0da --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -0,0 +1,123 @@ + + + + + 4.0.0 + + + org.apache.druid + druid + 0.17.0-incubating-SNAPSHOT + ../../pom.xml + + + org.apache.druid.extensions.contrib + dropwizard-emitter + dropwizard-emitter + Druid emitter extension to convert druid metric to Dropwizard metrics + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-server + ${project.parent.version} + provided + + + io.dropwizard.metrics + metrics-core + ${dropwizard.metrics.version} + + + io.dropwizard.metrics + metrics-jmx + ${dropwizard.metrics.version} + + + junit + junit + test + + + org.easymock + easymock + test + + + pl.pragmatists + JUnitParams + 1.0.4 + test + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + org.apache.curator + curator-client + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.github.ben-manes.caffeine + caffeine + provided + + + com.google.guava + guava + provided + + + com.google.code.findbugs + jsr305 + provided + + + diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java new file mode 100644 index 00000000000..7c1194726b3 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import org.apache.curator.shaded.com.google.common.io.Closeables; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.logger.Logger; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +/** + * + */ +public class DropwizardConverter +{ + private static final Logger log = new Logger(DropwizardConverter.class); + private final Map metricMap; + + public DropwizardConverter(ObjectMapper mapper, String dimensionMapPath) + { + metricMap = readMap(mapper, dimensionMapPath); + } + + /** + * Filters user dimensions for given metric and adds them to filteredDimensions. + * Returns null if there is no mapping present for the given metric. + */ + @Nullable + public DropwizardMetricSpec addFilteredUserDims( + String service, + String metric, + Map userDims, + Map filteredDimensions + ) + { + + // Find the metric in the map. If we cant find it try to look it up prefixed by the service name. + // This is because some metrics are reported differently, but with the same name, from different services. + DropwizardMetricSpec metricSpec = null; + DropwizardMetricSpec dropwizardMetricSpec = metricMap.get(metric); + if (dropwizardMetricSpec != null) { + metricSpec = dropwizardMetricSpec; + } else if (metricMap.containsKey(service + "-" + metric)) { + metricSpec = metricMap.get(service + "-" + metric); + } + if (metricSpec != null) { + for (String dim : metricSpec.getDimensions()) { + if (userDims.containsKey(dim)) { + filteredDimensions.put(dim, userDims.get(dim).toString()); + } + } + return metricSpec; + } else { + // No mapping found for given metric, return null + return null; + } + } + + private Map readMap(ObjectMapper mapper, String dimensionMapPath) + { + InputStream is = null; + try { + if (Strings.isNullOrEmpty(dimensionMapPath)) { + log.info("Using default metric dimension and types"); + is = this.getClass().getClassLoader().getResourceAsStream("defaultMetricDimensions.json"); + } else { + log.info("Using metric dimensions at types at [%s]", dimensionMapPath); + is = new FileInputStream(new File(dimensionMapPath)); + } + return mapper.readerFor(new TypeReference>() + { + }).readValue(is); + } + catch (IOException e) { + throw new ISE(e, "Failed to parse metric dimensions and types"); + } + finally { + Closeables.closeQuietly(is); + } + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java new file mode 100644 index 00000000000..5baa1b5da24 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.core.Emitter; +import org.apache.druid.java.util.emitter.core.Event; +import org.apache.druid.java.util.emitter.service.AlertEvent; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class DropwizardEmitter implements Emitter +{ + private static final Logger log = new Logger(DropwizardEmitter.class); + private final MetricRegistry metricsRegistry; + private final AtomicBoolean started = new AtomicBoolean(false); + private final DropwizardConverter converter; + private final List alertEmitters; + private final List reporters; + private final DropwizardEmitterConfig config; + + public DropwizardEmitter( + DropwizardEmitterConfig config, + ObjectMapper mapper, + List alertEmitters + ) + { + this.alertEmitters = alertEmitters; + this.config = config; + this.reporters = config.getReporters(); + this.converter = new DropwizardConverter(mapper, config.getDimensionMapPath()); + final Cache metricsRegistryCache = Caffeine.newBuilder() + .recordStats() + .maximumSize(config.getMaxMetricsRegistrySize()) + .build(); + metricsRegistry = new MetricRegistry() + { + @Override + protected ConcurrentMap buildMap() + { + return metricsRegistryCache.asMap(); + } + }; + } + + + @Override + public void start() + { + final boolean alreadyStarted = started.getAndSet(true); + if (!alreadyStarted) { + for (DropwizardReporter reporter : reporters) { + reporter.start(metricsRegistry); + } + } + } + + @Override + public void emit(Event event) + { + synchronized (started) { + if (!started.get()) { + throw new RejectedExecutionException("Dropwizard emitter Service not started."); + } + } + if (event instanceof ServiceMetricEvent) { + ServiceMetricEvent metricEvent = (ServiceMetricEvent) event; + String host = metricEvent.getHost(); + String service = metricEvent.getService(); + String metric = metricEvent.getMetric(); + Map userDims = metricEvent.getUserDims(); + Number value = metricEvent.getValue(); + ImmutableList.Builder nameBuilder = new ImmutableList.Builder<>(); + LinkedHashMap dims = new LinkedHashMap<>(); + final DropwizardMetricSpec metricSpec = converter.addFilteredUserDims(service, metric, userDims, dims); + + if (metricSpec != null) { + if (config.getPrefix() != null) { + nameBuilder.add(config.getPrefix()); + } + nameBuilder.add(StringUtils.format("metric=%s", metric)); + nameBuilder.add(StringUtils.format("service=%s", service)); + if (config.getIncludeHost()) { + nameBuilder.add(StringUtils.format("hostname=%s", host)); + } + dims.forEach((key, value1) -> nameBuilder.add(StringUtils.format("%s=%s", key, value1))); + + String fullName = StringUtils.replaceChar(Joiner.on(",").join(nameBuilder.build()), '/', "."); + updateMetric(fullName, value, metricSpec); + } else { + log.debug("Service=[%s], Metric=[%s] has no mapping", service, metric); + } + } else if (event instanceof AlertEvent) { + for (Emitter emitter : alertEmitters) { + emitter.emit(event); + } + } else { + throw new ISE("unknown event type [%s]", event.getClass()); + } + } + + private void updateMetric(String name, Number value, DropwizardMetricSpec metricSpec) + { + switch (metricSpec.getType()) { + case meter: + metricsRegistry.meter(name).mark(value.longValue()); + break; + case timer: + metricsRegistry.timer(name) + .update(value.longValue(), metricSpec.getTimeUnit()); + break; + case counter: + metricsRegistry.counter(name).inc(value.longValue()); + break; + case histogram: + metricsRegistry.histogram(name).update(value.longValue()); + break; + case gauge: + SettableGauge gauge = (SettableGauge) metricsRegistry.gauge(name, () -> new SettableGauge(value)); + gauge.setValue(value); + break; + default: + throw new ISE("Unknown Metric Type [%s]", metricSpec.getType()); + } + } + + @Override + public void flush() + { + for (DropwizardReporter reporter : reporters) { + reporter.flush(); + } + } + + @Override + public void close() + { + final boolean wasStarted = started.getAndSet(false); + if (wasStarted) { + for (DropwizardReporter reporter : reporters) { + reporter.close(); + } + } + } + + private static class SettableGauge implements Gauge + { + private Number value; + + public SettableGauge(Number value) + { + this.value = value; + } + + public void setValue(Number value) + { + this.value = value; + } + + @Override + public Number getValue() + { + return value; + } + } + +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java new file mode 100644 index 00000000000..c724df10f7f --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + + +public class DropwizardEmitterConfig +{ + // default to 100 Mb + private static int DEFAULT_METRICS_REGISTRY_SIZE = 100_000_000; + @JsonProperty + private final List reporters; + @JsonProperty + private final String prefix; + @JsonProperty + private final Boolean includeHost; + @JsonProperty + private final String dimensionMapPath; + @JsonProperty + private final List alertEmitters; + @JsonProperty + private final int maxMetricsRegistrySize; + + @JsonCreator + public DropwizardEmitterConfig( + @JsonProperty("reporters") List reporters, + @JsonProperty("prefix") String prefix, + @JsonProperty("includeHost") Boolean includeHost, + @JsonProperty("dimensionMapPath") String dimensionMapPath, + @JsonProperty("alertEmitters") List alertEmitters, + @JsonProperty("metricsRegistrySize") Integer maxMetricsRegistrySize + ) + { + Preconditions.checkArgument(reporters != null && !reporters.isEmpty()); + this.reporters = reporters; + this.prefix = prefix; + this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; + this.includeHost = includeHost != null ? includeHost : true; + this.dimensionMapPath = dimensionMapPath; + this.maxMetricsRegistrySize = maxMetricsRegistrySize == null ? DEFAULT_METRICS_REGISTRY_SIZE : maxMetricsRegistrySize; + } + + @JsonProperty + public List getReporters() + { + return reporters; + } + + @JsonProperty + public String getPrefix() + { + return prefix; + } + + @JsonProperty + public Boolean getIncludeHost() + { + return includeHost; + } + + @JsonProperty + public String getDimensionMapPath() + { + return dimensionMapPath; + } + + @JsonProperty + public List getAlertEmitters() + { + return alertEmitters; + } + + @JsonProperty + public int getMaxMetricsRegistrySize() + { + return maxMetricsRegistrySize; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; + return maxMetricsRegistrySize == that.maxMetricsRegistrySize && + Objects.equals(reporters, that.reporters) && + Objects.equals(prefix, that.prefix) && + Objects.equals(includeHost, that.includeHost) && + Objects.equals(dimensionMapPath, that.dimensionMapPath) && + Objects.equals(alertEmitters, that.alertEmitters); + } + + @Override + public int hashCode() + { + return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters, maxMetricsRegistrySize); + } + + @Override + public String toString() + { + return "DropwizardEmitterConfig{" + + "reporters=" + reporters + + ", prefix='" + prefix + '\'' + + ", includeHost=" + includeHost + + ", dimensionMapPath='" + dimensionMapPath + '\'' + + ", alertEmitters=" + alertEmitters + + ", maxMetricsRegistrySize=" + maxMetricsRegistrySize + + '}'; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java new file mode 100644 index 00000000000..a700b88050c --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.java.util.emitter.core.Emitter; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class DropwizardEmitterModule implements DruidModule +{ + private static final String EMITTER_TYPE = "dropwizard"; + + @Override + public List getJacksonModules() + { + return Collections.EMPTY_LIST; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter." + EMITTER_TYPE, DropwizardEmitterConfig.class); + } + + @Provides + @Named(EMITTER_TYPE) + public Emitter getEmitter( + DropwizardEmitterConfig dropwizardEmitterConfig, + ObjectMapper mapper, + final Injector injector + ) + { + List alertEmitters = dropwizardEmitterConfig.getAlertEmitters() + .stream() + .map(s -> injector.getInstance( + Key.get( + Emitter.class, + Names.named(s) + ))) + .collect(Collectors.toList()); + + return new DropwizardEmitter(dropwizardEmitterConfig, mapper, alertEmitters); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java new file mode 100644 index 00000000000..98a65292810 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class DropwizardMetricSpec +{ + @JsonProperty("dimensions") + private final List dimensions; + @JsonProperty("type") + private final Type type; + @JsonProperty("timeUnit") + private final TimeUnit timeUnit; + + @JsonCreator + DropwizardMetricSpec( + @JsonProperty("dimensions") List dimensions, + @JsonProperty("type") Type type, + @JsonProperty("timeUnit") TimeUnit timeUnit + ) + { + this.dimensions = dimensions; + this.type = type; + this.timeUnit = timeUnit; + } + + @JsonProperty + public Type getType() + { + return type; + } + + @JsonProperty + public List getDimensions() + { + return dimensions; + } + + @JsonProperty + public TimeUnit getTimeUnit() + { + return timeUnit; + } + + public enum Type + { + histogram, timer, meter, counter, gauge + } + +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java new file mode 100644 index 00000000000..9bb32481460 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "console", value = DropwizardConsoleReporter.class), + @JsonSubTypes.Type(name = "jmx", value = DropwizardJMXReporter.class), +}) +public interface DropwizardReporter +{ + + void start(MetricRegistry metricRegistry); + + /** + * Used for reporters that choose to buffer events to trigger flushing of buffered events. + * It should be a non-blocking operation. + */ + void flush(); + + void close(); +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java new file mode 100644 index 00000000000..0f465f8f710 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import java.util.LinkedHashMap; +import java.util.Map; + +class GaugesCache extends LinkedHashMap +{ + private int capacity; + + public GaugesCache(int capacity) + { + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) + { + return this.size() > this.capacity; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java new file mode 100644 index 00000000000..9b0a072faf0 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.emitter.dropwizard.DropwizardReporter; + +import java.util.concurrent.TimeUnit; + +public class DropwizardConsoleReporter implements DropwizardReporter +{ + private long emitIntervalInSecs; + private TimeUnit rates = TimeUnit.SECONDS; + private TimeUnit durations = TimeUnit.MILLISECONDS; + private ConsoleReporter consoleReporter; + + @JsonProperty + public long getEmitIntervalInSecs() + { + return emitIntervalInSecs; + } + + @JsonProperty + public void setEmitIntervalInSecs(long emitIntervalInSecs) + { + this.emitIntervalInSecs = emitIntervalInSecs; + } + + @JsonProperty + public TimeUnit getRates() + { + return rates; + } + + @JsonProperty + public void setRates(String rates) + { + this.rates = TimeUnit.valueOf(rates); + } + + @JsonProperty + public TimeUnit getDurations() + { + return durations; + } + + @JsonProperty + public void setDurations(String durations) + { + this.durations = TimeUnit.valueOf(durations); + } + + @Override + public void start(MetricRegistry metricRegistry) + { + consoleReporter = ConsoleReporter.forRegistry(metricRegistry) + .convertDurationsTo(durations) + .convertRatesTo(rates) + .build(); + consoleReporter.start(emitIntervalInSecs, TimeUnit.SECONDS); + + } + + @Override + public void flush() + { + // no-op + } + + @Override + public void close() + { + consoleReporter.stop(); + } + + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DropwizardConsoleReporter that = (DropwizardConsoleReporter) o; + + if (emitIntervalInSecs != that.emitIntervalInSecs) { + return false; + } + if (consoleReporter != null ? !consoleReporter.equals(that.consoleReporter) : that.consoleReporter != null) { + return false; + } + if (durations != that.durations) { + return false; + } + if (rates != that.rates) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = (int) (emitIntervalInSecs ^ (emitIntervalInSecs >>> 32)); + result = 31 * result + (rates != null ? rates.hashCode() : 0); + result = 31 * result + (durations != null ? durations.hashCode() : 0); + result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); + return result; + } + + @Override + public String toString() + { + return "DropwizardConsoleReporter{" + + "emitIntervalInSecs=" + emitIntervalInSecs + + ", rates=" + rates + + ", durations=" + durations + + '}'; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java new file mode 100644 index 00000000000..654ff67a5d1 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.jmx.JmxReporter; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.emitter.dropwizard.DropwizardReporter; + +import java.util.Objects; + +public class DropwizardJMXReporter implements DropwizardReporter +{ + private String domain = "org.apache.druid"; + private JmxReporter reporter; + + @JsonProperty + public String getDomain() + { + return domain; + } + + @Override + public void start(MetricRegistry metricRegistry) + { + reporter = JmxReporter.forRegistry(metricRegistry) + .inDomain(domain).build(); + reporter.start(); + } + + @Override + public void flush() + { + // no-op + } + + @Override + public void close() + { + reporter.close(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DropwizardJMXReporter that = (DropwizardJMXReporter) o; + return Objects.equals(domain, that.domain); + } + + @Override + public int hashCode() + { + return Objects.hash(domain); + } + + @Override + public String toString() + { + return "DropwizardJMXReporter{" + + "domain='" + domain + '\'' + + '}'; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 00000000000..c6d84260ee3 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.emitter.dropwizard.DropwizardEmitterModule diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json new file mode 100644 index 00000000000..ed071968a4f --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json @@ -0,0 +1,540 @@ +{ + "query/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/time": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/ttfb": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/backpressure": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/intervalChunk/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/segment/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/wait/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/scan/pending": { + "dimensions": [], + "type": "gauge" + }, + "query/segmentAndCache/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/cpu/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "query/cache/delta/numEntries": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/sizeBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hits": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/misses": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/evictions": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hitRate": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/averageBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/timeouts": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/errors": { + "dimensions": [], + "type": "counter" + }, + "query/cache/total/numEntries": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/sizeBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hits": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/misses": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/evictions": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hitRate": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/averageBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/timeouts": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/errors": { + "dimensions": [], + "type": "gauge" + }, + "ingest/events/thrownAway": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/unparseable": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/duplicate": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/processed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/rows/output": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/counter": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/persist/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/persist/backPressure": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "ingest/persist/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/handoff/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/merge/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/merge/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "task/run/time": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/added/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/moved/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/nuked/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/assigned/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/moved/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/dropped/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/deleted/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/unneeded/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/raw": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalization": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalized": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/loadQueue/size": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/failed": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/dropQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/size": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "segment/overShadowed/counter": { + "dimensions": [], + "type": "gauge" + }, + "segment/max": { + "dimensions": [], + "type": "gauge" + }, + "segment/used": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "segment/usedPercent": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "jvm/pool/committed": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/init": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/max": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/used": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/counter": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/used": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/capacity": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/mem/init": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/max": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/used": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/committed": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/gc/counter": { + "dimensions": [ + "gcName" + ], + "type": "counter" + }, + "jvm/gc/cpu": { + "dimensions": [ + "gcName" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/events/buffered": { + "dimensions": [ + "serviceName, bufferCapacity" + ], + "type": "gauge" + }, + "sys/swap/free": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageIn": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageOut": { + "dimensions": [], + "type": "gauge" + }, + "sys/disk/write/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/write/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/net/write/size": { + "dimensions": [], + "type": "counter" + }, + "sys/net/read/size": { + "dimensions": [], + "type": "counter" + }, + "sys/fs/used": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/fs/max": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/mem/used": { + "dimensions": [], + "type": "gauge" + }, + "sys/mem/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/storage/used": { + "dimensions": [ + "fsDirName" + ], + "type": "gauge" + }, + "sys/cpu": { + "dimensions": [ + "cpuName", + "cpuTime" + ], + "type": "gauge" + }, + "coordinator-segment/counter": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "historical-segment/counter": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + } +} \ No newline at end of file diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java new file mode 100644 index 00000000000..ed9b9f80931 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class DropwizardEmitterConfigTest +{ + private ObjectMapper mapper = new DefaultObjectMapper(); + + @Before + public void setUp() + { + mapper.setInjectableValues(new InjectableValues.Std().addValue( + ObjectMapper.class, + new DefaultObjectMapper() + )); + } + + @Test + public void testSerDeserDropwizardEmitterConfig() throws IOException + { + DropwizardEmitterConfig dropwizardEmitterConfig = new DropwizardEmitterConfig( + Lists.newArrayList(new DropwizardConsoleReporter(), new DropwizardJMXReporter()), + "my-prefix", + false, + "my/config/path", + null, + 400 + ); + String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); + DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.readerFor(DropwizardEmitterConfig.class).readValue( + dropwizardEmitterConfigString + ); + Assert.assertEquals(dropwizardEmitterConfigExpected, dropwizardEmitterConfig); + } + +} + + diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index c17439cbf8f..36a3f8c5814 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -18,7 +18,7 @@ ~ under the License. --> - 4.0.0 diff --git a/pom.xml b/pom.xml index 89dad996b7f..64ca5559c40 100644 --- a/pom.xml +++ b/pom.xml @@ -157,6 +157,7 @@ extensions-contrib/influx-extensions extensions-contrib/azure-extensions extensions-contrib/cassandra-storage + extensions-contrib/dropwizard-emitter extensions-contrib/cloudfiles-extensions extensions-contrib/graphite-emitter extensions-contrib/distinctcount diff --git a/website/.spelling b/website/.spelling index cae3b13b2b6..3167f85d1f3 100644 --- a/website/.spelling +++ b/website/.spelling @@ -54,6 +54,8 @@ Double.NEGATIVE_INFINITY Double.NEGATIVE_INFINITY. Double.POSITIVE_INFINITY Double.POSITIVE_INFINITY. +Dropwizard +dropwizard DruidSQL EC2 EMR @@ -90,6 +92,7 @@ JDK JDK7 JDK8 JKS +JMX JRE JS JSON @@ -760,6 +763,7 @@ statsd-emitter coords dimName maxCoords +Mb minCoords - ../docs/development/javascript.md Metaspace