HDFS-12502. nntop should support a category based on FilesInGetListingOps.
This commit is contained in:
parent
b016f08f67
commit
60bfee270e
|
@ -3673,6 +3673,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
readUnlock(operationName);
|
readUnlock(operationName);
|
||||||
}
|
}
|
||||||
logAuditEvent(true, operationName, src);
|
logAuditEvent(true, operationName, src);
|
||||||
|
if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation()
|
||||||
|
&& dl != null && Server.getRemoteUser() != null) {
|
||||||
|
topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(),
|
||||||
|
dl.getPartialListing().length);
|
||||||
|
}
|
||||||
return dl;
|
return dl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -70,6 +70,14 @@ public class TopMetrics implements MetricsSource {
|
||||||
public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
|
public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
|
||||||
public static final String TOPMETRICS_METRICS_SOURCE_NAME =
|
public static final String TOPMETRICS_METRICS_SOURCE_NAME =
|
||||||
"NNTopUserOpCounts";
|
"NNTopUserOpCounts";
|
||||||
|
/**
|
||||||
|
* In addition to counts of different RPC calls, NNTop also reports top
|
||||||
|
* users listing large directories (measured by the number of files involved
|
||||||
|
* in listing operations from the user). This is important because the CPU
|
||||||
|
* and GC overhead of a listing operation grows linearly with the number of
|
||||||
|
* files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}.
|
||||||
|
*/
|
||||||
|
public static final String FILES_IN_GETLISTING = "filesInGetListing";
|
||||||
private final boolean isMetricsSourceEnabled;
|
private final boolean isMetricsSourceEnabled;
|
||||||
|
|
||||||
private static void logConf(Configuration conf) {
|
private static void logConf(Configuration conf) {
|
||||||
|
@ -123,22 +131,30 @@ public class TopMetrics implements MetricsSource {
|
||||||
public void report(boolean succeeded, String userName, InetAddress addr,
|
public void report(boolean succeeded, String userName, InetAddress addr,
|
||||||
String cmd, String src, String dst, FileStatus status) {
|
String cmd, String src, String dst, FileStatus status) {
|
||||||
// currently nntop only makes use of the username and the command
|
// currently nntop only makes use of the username and the command
|
||||||
report(userName, cmd);
|
report(userName, cmd, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void report(String userName, String cmd) {
|
public void reportFilesInGetListing(String userName, int numFiles) {
|
||||||
|
report(userName, FILES_IN_GETLISTING, numFiles);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void report(String userName, String cmd, int delta) {
|
||||||
long currTime = Time.monotonicNow();
|
long currTime = Time.monotonicNow();
|
||||||
report(currTime, userName, cmd);
|
report(currTime, userName, cmd, delta);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void report(long currTime, String userName, String cmd) {
|
public void report(long currTime, String userName, String cmd, int delta) {
|
||||||
LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
|
LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
|
||||||
userName = UserGroupInformation.trimLoginMethod(userName);
|
userName = UserGroupInformation.trimLoginMethod(userName);
|
||||||
for (RollingWindowManager rollingWindowManager : rollingWindowManagers
|
for (RollingWindowManager rollingWindowManager : rollingWindowManagers
|
||||||
.values()) {
|
.values()) {
|
||||||
rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
|
rollingWindowManager.recordMetric(currTime, cmd, userName, delta);
|
||||||
rollingWindowManager.recordMetric(currTime,
|
// Increase the number of all RPC calls by the user, unless the report
|
||||||
TopConf.ALL_CMDS, userName, 1);
|
// is for the number of files in a listing operation.
|
||||||
|
if (!cmd.equals(FILES_IN_GETLISTING)) {
|
||||||
|
rollingWindowManager.recordMetric(currTime,
|
||||||
|
TopConf.ALL_CMDS, userName, delta);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.lib.Interns;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
|
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
|
||||||
|
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING;
|
||||||
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
|
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
|
||||||
import static org.mockito.Mockito.times;
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
|
@ -41,9 +42,10 @@ public class TestTopMetrics {
|
||||||
TopMetrics topMetrics = new TopMetrics(conf,
|
TopMetrics topMetrics = new TopMetrics(conf,
|
||||||
topConf.nntopReportingPeriodsMs);
|
topConf.nntopReportingPeriodsMs);
|
||||||
// Dummy command
|
// Dummy command
|
||||||
topMetrics.report("test", "listStatus");
|
topMetrics.report("test", "listStatus", 1);
|
||||||
topMetrics.report("test", "listStatus");
|
topMetrics.report("test", "listStatus", 1);
|
||||||
topMetrics.report("test", "listStatus");
|
topMetrics.report("test", "listStatus", 1);
|
||||||
|
topMetrics.report("test", FILES_IN_GETLISTING, 1000);
|
||||||
|
|
||||||
MetricsRecordBuilder rb = getMetrics(topMetrics);
|
MetricsRecordBuilder rb = getMetrics(topMetrics);
|
||||||
MetricsCollector mc = rb.parent();
|
MetricsCollector mc = rb.parent();
|
||||||
|
@ -59,5 +61,8 @@ public class TestTopMetrics {
|
||||||
|
|
||||||
verify(rb, times(3)).addCounter(Interns.info("op=listStatus." +
|
verify(rb, times(3)).addCounter(Interns.info("op=listStatus." +
|
||||||
"user=test.count", "Total operations performed by user"), 3L);
|
"user=test.count", "Total operations performed by user"), 3L);
|
||||||
|
|
||||||
|
verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING +
|
||||||
|
".user=test.count", "Total operations performed by user"), 1000L);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue