Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Vinayakumar B, Yongjun Zhang, and Brahma Reddy Battula. (ozawa)

This commit is contained in:
Tsuyoshi Ozawa 2016-07-13 21:28:04 +09:00
parent 438b7c5935
commit 77ffe76212
1 changed files with 14 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
View File

@ -31,6 +31,7 @@ import javax.management.ReflectionException;
import static com.google.common.base.Preconditions.*; import static com.google.common.base.Preconditions.*;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -59,7 +60,6 @@ class MetricsSourceAdapter implements DynamicMBean {
private final MBeanInfoBuilder infoBuilder; private final MBeanInfoBuilder infoBuilder;
private final Iterable<MetricsTag> injectedTags; private final Iterable<MetricsTag> injectedTags;
private Iterable<MetricsRecordImpl> lastRecs;
private boolean lastRecsCleared; private boolean lastRecsCleared;
private long jmxCacheTS = 0; private long jmxCacheTS = 0;
private long jmxCacheTTL; private long jmxCacheTTL;
@ -175,18 +175,19 @@ class MetricsSourceAdapter implements DynamicMBean {
} }
} }
// HADOOP-11361: Release lock here for avoid deadlock between
// MetricsSystemImpl's lock and MetricsSourceAdapter's lock.
Iterable<MetricsRecordImpl> lastRecs = null;
if (getAllMetrics) { if (getAllMetrics) {
MetricsCollectorImpl builder = new MetricsCollectorImpl(); lastRecs = getMetrics(new MetricsCollectorImpl(), true);
getMetrics(builder, true);
} }
synchronized(this) { synchronized (this) {
updateAttrCache(); if (lastRecs != null) {
if (getAllMetrics) { updateAttrCache(lastRecs);
updateInfoCache(); updateInfoCache(lastRecs);
} }
jmxCacheTS = Time.now(); jmxCacheTS = Time.now();
lastRecs = null; // in case regular interval update is not running
lastRecsCleared = true; lastRecsCleared = true;
} }
} }
@ -194,11 +195,6 @@ class MetricsSourceAdapter implements DynamicMBean {
Iterable<MetricsRecordImpl> getMetrics(MetricsCollectorImpl builder, Iterable<MetricsRecordImpl> getMetrics(MetricsCollectorImpl builder,
boolean all) { boolean all) {
builder.setRecordFilter(recordFilter).setMetricFilter(metricFilter); builder.setRecordFilter(recordFilter).setMetricFilter(metricFilter);
synchronized(this) {
if (lastRecs == null && jmxCacheTS == 0) {
all = true; // Get all the metrics to populate the sink caches
}
}
try { try {
source.getMetrics(builder, all); source.getMetrics(builder, all);
} catch (Exception e) { } catch (Exception e) {
@ -209,10 +205,7 @@ class MetricsSourceAdapter implements DynamicMBean {
rb.add(t); rb.add(t);
} }
} }
synchronized(this) { return builder.getRecords();
lastRecs = builder.getRecords();
return lastRecs;
}
} }
synchronized void stop() { synchronized void stop() {
@ -246,13 +239,15 @@ class MetricsSourceAdapter implements DynamicMBean {
return jmxCacheTTL; return jmxCacheTTL;
} }
private void updateInfoCache() { private void updateInfoCache(Iterable<MetricsRecordImpl> lastRecs) {
Preconditions.checkNotNull(lastRecs, "LastRecs should not be null");
LOG.debug("Updating info cache..."); LOG.debug("Updating info cache...");
infoCache = infoBuilder.reset(lastRecs).get(); infoCache = infoBuilder.reset(lastRecs).get();
LOG.debug("Done"); LOG.debug("Done");
} }
private int updateAttrCache() { private int updateAttrCache(Iterable<MetricsRecordImpl> lastRecs) {
Preconditions.checkNotNull(lastRecs, "LastRecs should not be null");
LOG.debug("Updating attr cache..."); LOG.debug("Updating attr cache...");
int recNo = 0; int recNo = 0;
int numMetrics = 0; int numMetrics = 0;