HADOOP-14503. Make RollingAverages a mutable metric. Contributed by Hanisha Koneru.
This commit is contained in:
parent
bec79ca249
commit
8633ef8e10
|
@ -313,6 +313,15 @@ public class MetricsRegistry {
|
||||||
return rates;
|
return rates;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized MutableRollingAverages newMutableRollingAverages(
|
||||||
|
String name, String valueName) {
|
||||||
|
checkMetricName(name);
|
||||||
|
MutableRollingAverages rollingAverages =
|
||||||
|
new MutableRollingAverages(valueName);
|
||||||
|
metricsMap.put(name, rollingAverages);
|
||||||
|
return rollingAverages;
|
||||||
|
}
|
||||||
|
|
||||||
synchronized void add(String name, MutableMetric metric) {
|
synchronized void add(String name, MutableMetric metric) {
|
||||||
checkMetricName(name);
|
checkMetricName(name);
|
||||||
metricsMap.put(name, metric);
|
metricsMap.put(name, metric);
|
||||||
|
|
|
@ -78,6 +78,10 @@ public class MutableMetricsFactory {
|
||||||
annotation.sampleName(), annotation.valueName(),
|
annotation.sampleName(), annotation.valueName(),
|
||||||
annotation.always());
|
annotation.always());
|
||||||
}
|
}
|
||||||
|
if (cls == MutableRollingAverages.class) {
|
||||||
|
return registry.newMutableRollingAverages(info.name(),
|
||||||
|
annotation.valueName());
|
||||||
|
}
|
||||||
throw new MetricsException("Unsupported metric field "+ field.getName() +
|
throw new MetricsException("Unsupported metric field "+ field.getName() +
|
||||||
" of type "+ field.getType().getName());
|
" of type "+ field.getType().getName());
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,6 +31,7 @@ import java.util.concurrent.ScheduledFuture;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
@ -57,29 +58,30 @@ import static org.apache.hadoop.metrics2.lib.Interns.*;
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class RollingAverages extends MutableMetric implements Closeable {
|
public class MutableRollingAverages extends MutableMetric implements Closeable {
|
||||||
|
|
||||||
private final MutableRatesWithAggregation innerMetrics =
|
private MutableRatesWithAggregation innerMetrics =
|
||||||
new MutableRatesWithAggregation();
|
new MutableRatesWithAggregation();
|
||||||
|
|
||||||
private static final ScheduledExecutorService SCHEDULER = Executors
|
@VisibleForTesting
|
||||||
|
static final ScheduledExecutorService SCHEDULER = Executors
|
||||||
.newScheduledThreadPool(1, new ThreadFactoryBuilder().setDaemon(true)
|
.newScheduledThreadPool(1, new ThreadFactoryBuilder().setDaemon(true)
|
||||||
.setNameFormat("RollingAverages-%d").build());
|
.setNameFormat("MutableRollingAverages-%d").build());
|
||||||
|
|
||||||
private ScheduledFuture<?> scheduledTask = null;
|
private ScheduledFuture<?> scheduledTask = null;
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
private Map<String, MutableRate> currentSnapshot;
|
private Map<String, MutableRate> currentSnapshot;
|
||||||
|
|
||||||
private final int numWindows;
|
|
||||||
private final String avgInfoNameTemplate;
|
private final String avgInfoNameTemplate;
|
||||||
private final String avgInfoDescTemplate;
|
private final String avgInfoDescTemplate;
|
||||||
|
private int numWindows;
|
||||||
|
|
||||||
private static class SumAndCount {
|
private static class SumAndCount {
|
||||||
private final double sum;
|
private final double sum;
|
||||||
private final long count;
|
private final long count;
|
||||||
|
|
||||||
public SumAndCount(final double sum, final long count) {
|
SumAndCount(final double sum, final long count) {
|
||||||
this.sum = sum;
|
this.sum = sum;
|
||||||
this.count = count;
|
this.count = count;
|
||||||
}
|
}
|
||||||
|
@ -105,44 +107,36 @@ public class RollingAverages extends MutableMetric implements Closeable {
|
||||||
private Map<String, LinkedBlockingDeque<SumAndCount>> averages =
|
private Map<String, LinkedBlockingDeque<SumAndCount>> averages =
|
||||||
new ConcurrentHashMap<>();
|
new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
private static final long WINDOW_SIZE_MS_DEFAULT = 300_000;
|
||||||
|
private static final int NUM_WINDOWS_DEFAULT = 36;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor of {@link RollingAverages}.
|
* Constructor for {@link MutableRollingAverages}.
|
||||||
* @param windowSizeMs
|
* @param metricValueName
|
||||||
* The number of milliseconds of each window for which subset
|
|
||||||
* of samples are gathered to compute the rolling average, A.K.A.
|
|
||||||
* roll over interval.
|
|
||||||
* @param numWindows
|
|
||||||
* The number of windows maintained to compute the rolling average.
|
|
||||||
* @param valueName
|
|
||||||
* of the metric (e.g. "Time", "Latency")
|
|
||||||
*/
|
*/
|
||||||
public RollingAverages(
|
public MutableRollingAverages(String metricValueName) {
|
||||||
final long windowSizeMs,
|
if (metricValueName == null) {
|
||||||
final int numWindows,
|
metricValueName = "";
|
||||||
final String valueName) {
|
}
|
||||||
String uvName = StringUtils.capitalize(valueName);
|
avgInfoNameTemplate = "[%s]" + "RollingAvg" +
|
||||||
String lvName = StringUtils.uncapitalize(valueName);
|
StringUtils.capitalize(metricValueName);
|
||||||
avgInfoNameTemplate = "[%s]" + "RollingAvg"+ uvName;
|
avgInfoDescTemplate = "Rolling average " +
|
||||||
avgInfoDescTemplate = "Rolling average "+ lvName +" for "+ "%s";
|
StringUtils.uncapitalize(metricValueName) +" for "+ "%s";
|
||||||
this.numWindows = numWindows;
|
numWindows = NUM_WINDOWS_DEFAULT;
|
||||||
scheduledTask = SCHEDULER.scheduleAtFixedRate(new RatesRoller(this),
|
scheduledTask = SCHEDULER.scheduleAtFixedRate(new RatesRoller(this),
|
||||||
windowSizeMs, windowSizeMs, TimeUnit.MILLISECONDS);
|
WINDOW_SIZE_MS_DEFAULT, WINDOW_SIZE_MS_DEFAULT, TimeUnit.MILLISECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor of {@link RollingAverages}.
|
* This method is for testing only to replace the scheduledTask.
|
||||||
* @param windowSizeMs
|
|
||||||
* The number of seconds of each window for which sub set of samples
|
|
||||||
* are gathered to compute rolling average, also A.K.A roll over
|
|
||||||
* interval.
|
|
||||||
* @param numWindows
|
|
||||||
* The number of windows maintained in the same time to compute the
|
|
||||||
* average of the rolling averages.
|
|
||||||
*/
|
*/
|
||||||
public RollingAverages(
|
@VisibleForTesting
|
||||||
final long windowSizeMs,
|
synchronized void replaceScheduledTask(int windows, long interval,
|
||||||
final int numWindows) {
|
TimeUnit timeUnit) {
|
||||||
this(windowSizeMs, numWindows, "Time");
|
numWindows = windows;
|
||||||
|
scheduledTask.cancel(true);
|
||||||
|
scheduledTask = SCHEDULER.scheduleAtFixedRate(new RatesRoller(this),
|
||||||
|
interval, interval, timeUnit);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -190,9 +184,9 @@ public class RollingAverages extends MutableMetric implements Closeable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class RatesRoller implements Runnable {
|
private static class RatesRoller implements Runnable {
|
||||||
private final RollingAverages parent;
|
private final MutableRollingAverages parent;
|
||||||
|
|
||||||
public RatesRoller(final RollingAverages parent) {
|
RatesRoller(final MutableRollingAverages parent) {
|
||||||
this.parent = parent;
|
this.parent = parent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -218,7 +212,7 @@ public class RollingAverages extends MutableMetric implements Closeable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Iterates over snapshot to capture all Avg metrics into rolling structure
|
* Iterates over snapshot to capture all Avg metrics into rolling structure
|
||||||
* {@link RollingAverages#averages}.
|
* {@link MutableRollingAverages#averages}.
|
||||||
*/
|
*/
|
||||||
private synchronized void rollOverAvgs() {
|
private synchronized void rollOverAvgs() {
|
||||||
if (currentSnapshot == null) {
|
if (currentSnapshot == null) {
|
||||||
|
@ -232,7 +226,7 @@ public class RollingAverages extends MutableMetric implements Closeable {
|
||||||
new Function<String, LinkedBlockingDeque<SumAndCount>>() {
|
new Function<String, LinkedBlockingDeque<SumAndCount>>() {
|
||||||
@Override
|
@Override
|
||||||
public LinkedBlockingDeque<SumAndCount> apply(String k) {
|
public LinkedBlockingDeque<SumAndCount> apply(String k) {
|
||||||
return new LinkedBlockingDeque<SumAndCount>(numWindows);
|
return new LinkedBlockingDeque<>(numWindows);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
final SumAndCount sumAndCount = new SumAndCount(
|
final SumAndCount sumAndCount = new SumAndCount(
|
|
@ -0,0 +1,49 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.metrics2.lib;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A helper class that can provide test cases access to package-private
|
||||||
|
* methods.
|
||||||
|
*/
|
||||||
|
public final class MetricsTestHelper {
|
||||||
|
public static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(MetricsTestHelper.class);
|
||||||
|
|
||||||
|
private MetricsTestHelper() {
|
||||||
|
//not called
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replace the rolling averages windows for a
|
||||||
|
* {@link MutableRollingAverages} metric.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public static void replaceRollingAveragesScheduler(
|
||||||
|
MutableRollingAverages mutableRollingAverages,
|
||||||
|
int numWindows, long interval, TimeUnit timeUnit) {
|
||||||
|
mutableRollingAverages.replaceScheduledTask(
|
||||||
|
numWindows, interval, timeUnit);
|
||||||
|
}
|
||||||
|
}
|
|
@ -17,23 +17,30 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.metrics2.lib;
|
package org.apache.hadoop.metrics2.lib;
|
||||||
|
|
||||||
|
import com.google.common.base.Supplier;
|
||||||
|
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
|
||||||
|
import org.apache.hadoop.metrics2.annotation.Metric;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
|
||||||
|
import org.apache.hadoop.util.Time;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.apache.hadoop.metrics2.lib.Interns.info;
|
import static org.apache.hadoop.metrics2.lib.Interns.info;
|
||||||
import static org.apache.hadoop.test.MetricsAsserts.mockMetricsRecordBuilder;
|
import static org.apache.hadoop.test.MetricsAsserts.*;
|
||||||
import static org.mockito.Mockito.times;
|
|
||||||
import static org.mockito.Mockito.never;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Matchers.anyDouble;
|
import static org.mockito.Matchers.anyDouble;
|
||||||
import static org.mockito.Matchers.eq;
|
import static org.mockito.Matchers.eq;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
|
|
||||||
import org.apache.hadoop.util.Time;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class tests various cases of the algorithms implemented in
|
* This class tests various cases of the algorithms implemented in
|
||||||
* {@link RollingAverages}.
|
* {@link MutableRollingAverages}.
|
||||||
*/
|
*/
|
||||||
public class TestRollingAverages {
|
public class TestMutableRollingAverages {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests if the results are correct if no samples are inserted, dry run of
|
* Tests if the results are correct if no samples are inserted, dry run of
|
||||||
* empty roll over.
|
* empty roll over.
|
||||||
|
@ -42,8 +49,9 @@ public class TestRollingAverages {
|
||||||
public void testRollingAveragesEmptyRollover() throws Exception {
|
public void testRollingAveragesEmptyRollover() throws Exception {
|
||||||
final MetricsRecordBuilder rb = mockMetricsRecordBuilder();
|
final MetricsRecordBuilder rb = mockMetricsRecordBuilder();
|
||||||
/* 5s interval and 2 windows */
|
/* 5s interval and 2 windows */
|
||||||
try (RollingAverages rollingAverages =
|
try (MutableRollingAverages rollingAverages =
|
||||||
new RollingAverages(5000, 2)) {
|
new MutableRollingAverages("Time")) {
|
||||||
|
rollingAverages.replaceScheduledTask(2, 5, TimeUnit.SECONDS);
|
||||||
/* Check it initially */
|
/* Check it initially */
|
||||||
rollingAverages.snapshot(rb, true);
|
rollingAverages.snapshot(rb, true);
|
||||||
verify(rb, never()).addGauge(
|
verify(rb, never()).addGauge(
|
||||||
|
@ -78,9 +86,9 @@ public class TestRollingAverages {
|
||||||
final int windowSizeMs = 5000; // 5s roll over interval
|
final int windowSizeMs = 5000; // 5s roll over interval
|
||||||
final int numWindows = 2;
|
final int numWindows = 2;
|
||||||
final int numOpsPerIteration = 1000;
|
final int numOpsPerIteration = 1000;
|
||||||
try (RollingAverages rollingAverages = new RollingAverages(windowSizeMs,
|
try (MutableRollingAverages rollingAverages =
|
||||||
numWindows)) {
|
new MutableRollingAverages("Time")) {
|
||||||
|
rollingAverages.replaceScheduledTask(2, 5000, TimeUnit.MILLISECONDS);
|
||||||
/* Push values for three intervals */
|
/* Push values for three intervals */
|
||||||
final long start = Time.monotonicNow();
|
final long start = Time.monotonicNow();
|
||||||
for (int i = 1; i <= 3; i++) {
|
for (int i = 1; i <= 3; i++) {
|
||||||
|
@ -121,4 +129,64 @@ public class TestRollingAverages {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that MutableRollingAverages gives expected results after
|
||||||
|
* initialization.
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test(timeout = 30000)
|
||||||
|
public void testMutableRollingAveragesMetric() throws Exception {
|
||||||
|
DummyTestMetric testMetric = new DummyTestMetric();
|
||||||
|
testMetric.create();
|
||||||
|
|
||||||
|
testMetric.add("metric1", 100);
|
||||||
|
testMetric.add("metric1", 900);
|
||||||
|
testMetric.add("metric2", 1000);
|
||||||
|
testMetric.add("metric2", 1000);
|
||||||
|
|
||||||
|
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
||||||
|
@Override
|
||||||
|
public Boolean get() {
|
||||||
|
testMetric.collectThreadLocalStates();
|
||||||
|
return testMetric.getStats().size() > 0;
|
||||||
|
}
|
||||||
|
}, 500, 5000);
|
||||||
|
|
||||||
|
MetricsRecordBuilder rb = getMetrics(DummyTestMetric.METRIC_NAME);
|
||||||
|
|
||||||
|
double metric1Avg = getDoubleGauge("[Metric1]RollingAvgTesting", rb);
|
||||||
|
double metric2Avg = getDoubleGauge("[Metric2]RollingAvgTesting", rb);
|
||||||
|
Assert.assertTrue("The rolling average of metric1 is not as expected",
|
||||||
|
metric1Avg == 500.0);
|
||||||
|
Assert.assertTrue("The rolling average of metric2 is not as expected",
|
||||||
|
metric2Avg == 1000.0);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
class DummyTestMetric {
|
||||||
|
@Metric (valueName = "testing")
|
||||||
|
private MutableRollingAverages rollingAverages;
|
||||||
|
|
||||||
|
static final String METRIC_NAME = "RollingAveragesTestMetric";
|
||||||
|
|
||||||
|
protected void create() {
|
||||||
|
DefaultMetricsSystem.instance().register(METRIC_NAME,
|
||||||
|
"mutable rolling averages test", this);
|
||||||
|
rollingAverages.replaceScheduledTask(10, 1000, TimeUnit.MILLISECONDS);
|
||||||
|
}
|
||||||
|
|
||||||
|
void add(String name, long latency) {
|
||||||
|
rollingAverages.add(name, latency);
|
||||||
|
}
|
||||||
|
|
||||||
|
void collectThreadLocalStates() {
|
||||||
|
rollingAverages.collectThreadLocalStates();
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Double> getStats() {
|
||||||
|
return rollingAverages.getStats(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -465,18 +465,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
||||||
HdfsClientConfigKeys.DeprecatedKeys.DFS_METRICS_SESSION_ID_KEY;
|
HdfsClientConfigKeys.DeprecatedKeys.DFS_METRICS_SESSION_ID_KEY;
|
||||||
public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY = "dfs.metrics.percentiles.intervals";
|
public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY = "dfs.metrics.percentiles.intervals";
|
||||||
|
|
||||||
// The following setting is not meant to be changed by administrators.
|
|
||||||
public static final String DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_KEY =
|
|
||||||
"dfs.metrics.rolling.averages.window.length";
|
|
||||||
public static final String DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_DEFAULT =
|
|
||||||
"5m";
|
|
||||||
|
|
||||||
// The following setting is not meant to be changed by administrators.
|
|
||||||
public static final String DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_KEY =
|
|
||||||
"dfs.metrics.rolling.average.num.windows";
|
|
||||||
public static final int DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_DEFAULT =
|
|
||||||
36;
|
|
||||||
|
|
||||||
public static final String DFS_DATANODE_PEER_STATS_ENABLED_KEY =
|
public static final String DFS_DATANODE_PEER_STATS_ENABLED_KEY =
|
||||||
"dfs.datanode.peer.stats.enabled";
|
"dfs.datanode.peer.stats.enabled";
|
||||||
public static final boolean DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT = false;
|
public static final boolean DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT = false;
|
||||||
|
|
|
@ -1415,7 +1415,7 @@ public class DataNode extends ReconfigurableBase
|
||||||
|
|
||||||
metrics = DataNodeMetrics.create(getConf(), getDisplayName());
|
metrics = DataNodeMetrics.create(getConf(), getDisplayName());
|
||||||
peerMetrics = dnConf.peerStatsEnabled ?
|
peerMetrics = dnConf.peerStatsEnabled ?
|
||||||
DataNodePeerMetrics.create(getConf(), getDisplayName()) : null;
|
DataNodePeerMetrics.create(getDisplayName()) : null;
|
||||||
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
|
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
|
||||||
|
|
||||||
ecWorker = new ErasureCodingWorker(getConf(), this);
|
ecWorker = new ErasureCodingWorker(getConf(), this);
|
||||||
|
|
|
@ -22,16 +22,13 @@ package org.apache.hadoop.hdfs.server.datanode.metrics;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.metrics2.MetricsJsonBuilder;
|
import org.apache.hadoop.metrics2.MetricsJsonBuilder;
|
||||||
import org.apache.hadoop.metrics2.lib.RollingAverages;
|
import org.apache.hadoop.metrics2.lib.MutableRollingAverages;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class maintains DataNode peer metrics (e.g. numOps, AvgTime, etc.) for
|
* This class maintains DataNode peer metrics (e.g. numOps, AvgTime, etc.) for
|
||||||
|
@ -44,7 +41,7 @@ public class DataNodePeerMetrics {
|
||||||
public static final Logger LOG = LoggerFactory.getLogger(
|
public static final Logger LOG = LoggerFactory.getLogger(
|
||||||
DataNodePeerMetrics.class);
|
DataNodePeerMetrics.class);
|
||||||
|
|
||||||
private final RollingAverages sendPacketDownstreamRollingAvgerages;
|
private final MutableRollingAverages sendPacketDownstreamRollingAverages;
|
||||||
|
|
||||||
private final String name;
|
private final String name;
|
||||||
|
|
||||||
|
@ -64,15 +61,11 @@ public class DataNodePeerMetrics {
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
static final long MIN_OUTLIER_DETECTION_SAMPLES = 1000;
|
static final long MIN_OUTLIER_DETECTION_SAMPLES = 1000;
|
||||||
|
|
||||||
public DataNodePeerMetrics(
|
public DataNodePeerMetrics(final String name) {
|
||||||
final String name,
|
|
||||||
final long windowSizeMs,
|
|
||||||
final int numWindows) {
|
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
|
this.slowNodeDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_NODES,
|
||||||
LOW_THRESHOLD_MS);
|
LOW_THRESHOLD_MS);
|
||||||
sendPacketDownstreamRollingAvgerages = new RollingAverages(
|
sendPacketDownstreamRollingAverages = new MutableRollingAverages("Time");
|
||||||
windowSizeMs, numWindows);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public String name() {
|
public String name() {
|
||||||
|
@ -82,23 +75,12 @@ public class DataNodePeerMetrics {
|
||||||
/**
|
/**
|
||||||
* Creates an instance of DataNodePeerMetrics, used for registration.
|
* Creates an instance of DataNodePeerMetrics, used for registration.
|
||||||
*/
|
*/
|
||||||
public static DataNodePeerMetrics create(Configuration conf, String dnName) {
|
public static DataNodePeerMetrics create(String dnName) {
|
||||||
final String name = "DataNodePeerActivity-" + (dnName.isEmpty()
|
final String name = "DataNodePeerActivity-" + (dnName.isEmpty()
|
||||||
? "UndefinedDataNodeName" + ThreadLocalRandom.current().nextInt()
|
? "UndefinedDataNodeName" + ThreadLocalRandom.current().nextInt()
|
||||||
: dnName.replace(':', '-'));
|
: dnName.replace(':', '-'));
|
||||||
|
|
||||||
final long windowSizeMs = conf.getTimeDuration(
|
return new DataNodePeerMetrics(name);
|
||||||
DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_KEY,
|
|
||||||
DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_DEFAULT,
|
|
||||||
TimeUnit.MILLISECONDS);
|
|
||||||
final int numWindows = conf.getInt(
|
|
||||||
DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_KEY,
|
|
||||||
DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_DEFAULT);
|
|
||||||
|
|
||||||
return new DataNodePeerMetrics(
|
|
||||||
name,
|
|
||||||
windowSizeMs,
|
|
||||||
numWindows);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -112,7 +94,7 @@ public class DataNodePeerMetrics {
|
||||||
public void addSendPacketDownstream(
|
public void addSendPacketDownstream(
|
||||||
final String peerAddr,
|
final String peerAddr,
|
||||||
final long elapsedMs) {
|
final long elapsedMs) {
|
||||||
sendPacketDownstreamRollingAvgerages.add(peerAddr, elapsedMs);
|
sendPacketDownstreamRollingAverages.add(peerAddr, elapsedMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -120,7 +102,7 @@ public class DataNodePeerMetrics {
|
||||||
*/
|
*/
|
||||||
public String dumpSendPacketDownstreamAvgInfoAsJson() {
|
public String dumpSendPacketDownstreamAvgInfoAsJson() {
|
||||||
final MetricsJsonBuilder builder = new MetricsJsonBuilder(null);
|
final MetricsJsonBuilder builder = new MetricsJsonBuilder(null);
|
||||||
sendPacketDownstreamRollingAvgerages.snapshot(builder, true);
|
sendPacketDownstreamRollingAverages.snapshot(builder, true);
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -128,7 +110,7 @@ public class DataNodePeerMetrics {
|
||||||
* Collects states maintained in {@link ThreadLocal}, if any.
|
* Collects states maintained in {@link ThreadLocal}, if any.
|
||||||
*/
|
*/
|
||||||
public void collectThreadLocalStates() {
|
public void collectThreadLocalStates() {
|
||||||
sendPacketDownstreamRollingAvgerages.collectThreadLocalStates();
|
sendPacketDownstreamRollingAverages.collectThreadLocalStates();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -139,10 +121,14 @@ public class DataNodePeerMetrics {
|
||||||
// This maps the metric name to the aggregate latency.
|
// This maps the metric name to the aggregate latency.
|
||||||
// The metric name is the datanode ID.
|
// The metric name is the datanode ID.
|
||||||
final Map<String, Double> stats =
|
final Map<String, Double> stats =
|
||||||
sendPacketDownstreamRollingAvgerages.getStats(
|
sendPacketDownstreamRollingAverages.getStats(
|
||||||
MIN_OUTLIER_DETECTION_SAMPLES);
|
MIN_OUTLIER_DETECTION_SAMPLES);
|
||||||
LOG.trace("DataNodePeerMetrics: Got stats: {}", stats);
|
LOG.trace("DataNodePeerMetrics: Got stats: {}", stats);
|
||||||
|
|
||||||
return slowNodeDetector.getOutliers(stats);
|
return slowNodeDetector.getOutliers(stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public MutableRollingAverages getSendPacketDownstreamRollingAverages() {
|
||||||
|
return sendPacketDownstreamRollingAverages;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||||
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodePeerMetrics;
|
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodePeerMetrics;
|
||||||
|
import org.apache.hadoop.metrics2.lib.MetricsTestHelper;
|
||||||
import org.apache.hadoop.util.Time;
|
import org.apache.hadoop.util.Time;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -42,16 +43,13 @@ public class TestDataNodePeerMetrics {
|
||||||
final int numOpsPerIteration = 1000;
|
final int numOpsPerIteration = 1000;
|
||||||
|
|
||||||
final Configuration conf = new HdfsConfiguration();
|
final Configuration conf = new HdfsConfiguration();
|
||||||
conf.setTimeDuration(
|
|
||||||
DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_KEY,
|
|
||||||
windowSize, TimeUnit.SECONDS);
|
|
||||||
conf.setInt(DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_KEY,
|
|
||||||
numWindows);
|
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, true);
|
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, true);
|
||||||
|
|
||||||
final DataNodePeerMetrics peerMetrics = DataNodePeerMetrics.create(
|
final DataNodePeerMetrics peerMetrics = DataNodePeerMetrics.create(
|
||||||
conf,
|
|
||||||
"Sample-DataNode");
|
"Sample-DataNode");
|
||||||
|
MetricsTestHelper.replaceRollingAveragesScheduler(
|
||||||
|
peerMetrics.getSendPacketDownstreamRollingAverages(),
|
||||||
|
numWindows, windowSize, TimeUnit.SECONDS);
|
||||||
final long start = Time.monotonicNow();
|
final long start = Time.monotonicNow();
|
||||||
for (int i = 1; i <= iterations; i++) {
|
for (int i = 1; i <= iterations; i++) {
|
||||||
final String peerAddr = genPeerAddress();
|
final String peerAddr = genPeerAddress();
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
package org.apache.hadoop.hdfs.server.datanode.metrics;
|
package org.apache.hadoop.hdfs.server.datanode.metrics;
|
||||||
|
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
|
import org.apache.hadoop.metrics2.lib.MetricsTestHelper;
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -30,6 +31,7 @@ import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.hamcrest.core.Is.is;
|
import static org.hamcrest.core.Is.is;
|
||||||
import static org.junit.Assert.assertThat;
|
import static org.junit.Assert.assertThat;
|
||||||
|
@ -73,8 +75,12 @@ public class TestDataNodeOutlierDetectionViaMetrics {
|
||||||
final String slowNodeName = "SlowNode";
|
final String slowNodeName = "SlowNode";
|
||||||
|
|
||||||
DataNodePeerMetrics peerMetrics = new DataNodePeerMetrics(
|
DataNodePeerMetrics peerMetrics = new DataNodePeerMetrics(
|
||||||
"PeerMetrics-For-Test", WINDOW_INTERVAL_SECONDS,
|
"PeerMetrics-For-Test");
|
||||||
ROLLING_AVERAGE_WINDOWS);
|
|
||||||
|
MetricsTestHelper.replaceRollingAveragesScheduler(
|
||||||
|
peerMetrics.getSendPacketDownstreamRollingAverages(),
|
||||||
|
ROLLING_AVERAGE_WINDOWS,
|
||||||
|
WINDOW_INTERVAL_SECONDS, TimeUnit.SECONDS);
|
||||||
|
|
||||||
injectFastNodesSamples(peerMetrics);
|
injectFastNodesSamples(peerMetrics);
|
||||||
injectSlowNodeSamples(peerMetrics, slowNodeName);
|
injectSlowNodeSamples(peerMetrics, slowNodeName);
|
||||||
|
@ -101,8 +107,12 @@ public class TestDataNodeOutlierDetectionViaMetrics {
|
||||||
@Test
|
@Test
|
||||||
public void testWithNoOutliers() throws Exception {
|
public void testWithNoOutliers() throws Exception {
|
||||||
DataNodePeerMetrics peerMetrics = new DataNodePeerMetrics(
|
DataNodePeerMetrics peerMetrics = new DataNodePeerMetrics(
|
||||||
"PeerMetrics-For-Test", WINDOW_INTERVAL_SECONDS,
|
"PeerMetrics-For-Test");
|
||||||
ROLLING_AVERAGE_WINDOWS);
|
|
||||||
|
MetricsTestHelper.replaceRollingAveragesScheduler(
|
||||||
|
peerMetrics.getSendPacketDownstreamRollingAverages(),
|
||||||
|
ROLLING_AVERAGE_WINDOWS,
|
||||||
|
WINDOW_INTERVAL_SECONDS, TimeUnit.SECONDS);
|
||||||
|
|
||||||
injectFastNodesSamples(peerMetrics);
|
injectFastNodesSamples(peerMetrics);
|
||||||
|
|
||||||
|
|
|
@ -66,10 +66,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
|
||||||
// Purposely hidden, based on comments in DFSConfigKeys
|
// Purposely hidden, based on comments in DFSConfigKeys
|
||||||
configurationPropsToSkipCompare
|
configurationPropsToSkipCompare
|
||||||
.add(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY);
|
.add(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY);
|
||||||
configurationPropsToSkipCompare
|
|
||||||
.add(DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGES_WINDOW_LENGTH_KEY);
|
|
||||||
configurationPropsToSkipCompare
|
|
||||||
.add(DFSConfigKeys.DFS_METRICS_ROLLING_AVERAGE_NUM_WINDOWS_KEY);
|
|
||||||
|
|
||||||
// Fully deprecated properties?
|
// Fully deprecated properties?
|
||||||
configurationPropsToSkipCompare
|
configurationPropsToSkipCompare
|
||||||
|
|
Loading…
Reference in New Issue