Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14235. Handle ArrayIndexOutOfBoundsException in DataNodeDiskMetrics#slowDiskDetectionDaemon. Contributed by Ranith Sardar.

This commit is contained in:
Surendra Singh Lilhore 2019-02-20 16:56:10 +05:30
parent 642fe6a260
commit 41e18feda3
1 changed files with 42 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
View File

@ -57,6 +57,10 @@ public class DataNodeDiskMetrics {
private volatile Map<String, Map<DiskOp, Double>> private volatile Map<String, Map<DiskOp, Double>>
diskOutliersStats = Maps.newHashMap(); diskOutliersStats = Maps.newHashMap();
// Adding for test purpose. When addSlowDiskForTesting() called from test
// code, status should not be overridden by daemon thread.
private boolean overrideStatus = true;
public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) { public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
this.dn = dn; this.dn = dn;
this.detectionInterval = diskOutlierDetectionIntervalMs; this.detectionInterval = diskOutlierDetectionIntervalMs;
@ -71,41 +75,43 @@ private void startDiskOutlierDetectionThread() {
@Override @Override
public void run() { public void run() {
while (shouldRun) { while (shouldRun) {
Map<String, Double> metadataOpStats = Maps.newHashMap(); if (dn.getFSDataset() != null) {
Map<String, Double> readIoStats = Maps.newHashMap(); Map<String, Double> metadataOpStats = Maps.newHashMap();
Map<String, Double> writeIoStats = Maps.newHashMap(); Map<String, Double> readIoStats = Maps.newHashMap();
FsDatasetSpi.FsVolumeReferences fsVolumeReferences = null; Map<String, Double> writeIoStats = Maps.newHashMap();
try { FsDatasetSpi.FsVolumeReferences fsVolumeReferences = null;
fsVolumeReferences = dn.getFSDataset().getFsVolumeReferences(); try {
Iterator<FsVolumeSpi> volumeIterator = fsVolumeReferences fsVolumeReferences = dn.getFSDataset().getFsVolumeReferences();
.iterator(); Iterator<FsVolumeSpi> volumeIterator = fsVolumeReferences
while (volumeIterator.hasNext()) { .iterator();
FsVolumeSpi volume = volumeIterator.next(); while (volumeIterator.hasNext()) {
DataNodeVolumeMetrics metrics = volumeIterator.next().getMetrics(); FsVolumeSpi volume = volumeIterator.next();
String volumeName = volume.getBaseURI().getPath(); DataNodeVolumeMetrics metrics = volume.getMetrics();
String volumeName = volume.getBaseURI().getPath();
metadataOpStats.put(volumeName, metadataOpStats.put(volumeName,
metrics.getMetadataOperationMean()); metrics.getMetadataOperationMean());
readIoStats.put(volumeName, metrics.getReadIoMean()); readIoStats.put(volumeName, metrics.getReadIoMean());
writeIoStats.put(volumeName, metrics.getWriteIoMean()); writeIoStats.put(volumeName, metrics.getWriteIoMean());
} }
} finally { } finally {
if (fsVolumeReferences != null) { if (fsVolumeReferences != null) {
try { try {
fsVolumeReferences.close(); fsVolumeReferences.close();
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error in releasing FS Volume references", e); LOG.error("Error in releasing FS Volume references", e);
}
} }
} }
} if (metadataOpStats.isEmpty() && readIoStats.isEmpty()
if (metadataOpStats.isEmpty() && readIoStats.isEmpty() && && writeIoStats.isEmpty()) {
writeIoStats.isEmpty()) { LOG.debug("No disk stats available for detecting outliers.");
LOG.debug("No disk stats available for detecting outliers."); continue;
return; }
}
detectAndUpdateDiskOutliers(metadataOpStats, readIoStats, detectAndUpdateDiskOutliers(metadataOpStats, readIoStats,
writeIoStats); writeIoStats);
}
try { try {
Thread.sleep(detectionInterval); Thread.sleep(detectionInterval);
@ -143,9 +149,10 @@ private void detectAndUpdateDiskOutliers(Map<String, Double> metadataOpStats,
for (Map.Entry<String, Double> entry : writeIoOutliers.entrySet()) { for (Map.Entry<String, Double> entry : writeIoOutliers.entrySet()) {
addDiskStat(diskStats, entry.getKey(), DiskOp.WRITE, entry.getValue()); addDiskStat(diskStats, entry.getKey(), DiskOp.WRITE, entry.getValue());
} }
if (overrideStatus) {
diskOutliersStats = diskStats; diskOutliersStats = diskStats;
LOG.debug("Updated disk outliers."); LOG.debug("Updated disk outliers.");
}
} }
private void addDiskStat(Map<String, Map<DiskOp, Double>> diskStats, private void addDiskStat(Map<String, Map<DiskOp, Double>> diskStats,
@ -176,6 +183,7 @@ public void shutdownAndWait() {
@VisibleForTesting @VisibleForTesting
public void addSlowDiskForTesting(String slowDiskPath, public void addSlowDiskForTesting(String slowDiskPath,
Map<DiskOp, Double> latencies) { Map<DiskOp, Double> latencies) {
overrideStatus = false;
if (latencies == null) { if (latencies == null) {
diskOutliersStats.put(slowDiskPath, ImmutableMap.of()); diskOutliersStats.put(slowDiskPath, ImmutableMap.of());
} else { } else {