HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru.
This commit is contained in:
Arpit Agarwal
parent
f597f4c43e
commit
959940b0ab
|
|
@ -332,7 +332,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
|
|||
FsVolume
|
||||
--------
|
||||
|
||||
Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics.
|
||||
Per-volume metrics contain Datanode Volume IO related statistics. Per-volume
|
||||
metrics are off by default. They can be enabled by setting `dfs.datanode
|
||||
.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0.
|
||||
Setting this value to 0.0 would mean profiling is not enabled. But enabling
|
||||
per-volume metrics may have a performance impact. Each metrics record
|
||||
contains tags such as Hostname as additional information along with metrics.
|
||||
|
||||
| Name | Description |
|
||||
|:---- |:---- |
|
||||
|
|
|
|||
|
|
@ -718,10 +718,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
|||
public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
|
||||
|
||||
// Datanode File IO Stats
|
||||
public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
|
||||
"dfs.datanode.enable.fileio.profiling";
|
||||
public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT =
|
||||
false;
|
||||
public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY =
|
||||
"dfs.datanode.enable.fileio.fault.injection";
|
||||
public static final boolean
|
||||
|
|
@ -730,7 +726,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
|||
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY =
|
||||
"dfs.datanode.fileio.profiling.sampling.fraction";
|
||||
public static final double
|
||||
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0;
|
||||
DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0;
|
||||
|
||||
//Keys with no defaults
|
||||
public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins";
|
||||
|
|
|
|||
|
|
@ -388,4 +388,21 @@ public final class Util {
|
|||
addrsList.removeAll(addrsToExclude);
|
||||
return addrsList;
|
||||
}
|
||||
|
||||
public static boolean isDiskStatsEnabled(double fileIOSamplingFraction) {
|
||||
final boolean isEnabled;
|
||||
if (fileIOSamplingFraction < 0.000001) {
|
||||
LOG.info(DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
|
||||
+ fileIOSamplingFraction + ". Disabling file IO profiling");
|
||||
isEnabled = false;
|
||||
} else {
|
||||
LOG.info(DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to "
|
||||
+ fileIOSamplingFraction + ". Enabling file IO profiling");
|
||||
isEnabled = true;
|
||||
}
|
||||
|
||||
return isEnabled;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
|
|||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
|
||||
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
|
||||
import org.apache.hadoop.hdfs.server.common.Util;
|
||||
import org.apache.hadoop.security.SaslPropertiesResolver;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
|
@ -174,9 +175,9 @@ public class DNConf {
|
|||
this.peerStatsEnabled = getConf().getBoolean(
|
||||
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
|
||||
this.diskStatsEnabled = getConf().getBoolean(
|
||||
DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
|
||||
this.diskStatsEnabled = Util.isDiskStatsEnabled(getConf().getDouble(
|
||||
DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT));
|
||||
this.outliersReportIntervalMs = getConf().getTimeDuration(
|
||||
DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
|
||||
DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*;
|
|||
*
|
||||
* Behavior can be injected into these events by enabling the
|
||||
* profiling and/or fault injection event hooks through
|
||||
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and
|
||||
* {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and
|
||||
* {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}.
|
||||
* These event hooks are disabled by default.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.server.common.Util;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
|
||||
import org.apache.hadoop.util.Time;
|
||||
|
|
@ -43,13 +44,11 @@ class ProfilingFileIoEvents {
|
|||
|
||||
public ProfilingFileIoEvents(@Nullable Configuration conf) {
|
||||
if (conf != null) {
|
||||
isEnabled = conf.getBoolean(DFSConfigKeys
|
||||
.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys
|
||||
.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT);
|
||||
double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
|
||||
double fileIOSamplingFraction = conf.getDouble(
|
||||
DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY,
|
||||
DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT);
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT);
|
||||
isEnabled = Util.isDiskStatsEnabled(fileIOSamplingFraction);
|
||||
if (fileIOSamplingFraction > 1) {
|
||||
LOG.warn(DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY +
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics {
|
|||
|
||||
private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException {
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
conf.setBoolean(DFSConfigKeys
|
||||
.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true);
|
||||
conf.setDouble(DFSConfigKeys
|
||||
.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
|
||||
SimulatedFSDataset.setFactory(conf);
|
||||
return new MiniDFSCluster.Builder(conf)
|
||||
.numDataNodes(NUM_DATANODES)
|
||||
|
|
|
|||
|
|
@ -103,8 +103,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
|
|||
.add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
|
||||
configurationPropsToSkipCompare
|
||||
.add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
|
||||
configurationPropsToSkipCompare
|
||||
.add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY);
|
||||
configurationPropsToSkipCompare.add(DFSConfigKeys
|
||||
.DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
|
||||
configurationPropsToSkipCompare.add(DFSConfigKeys
|
||||
|
|
|
|||
Loading…
Reference in New Issue