diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index f9647359d73..f8bc95b9a54 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -276,6 +276,7 @@ The following table lists the configuration property names that are deprecated i
| user.name | mapreduce.job.user.name |
| webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted |
| yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts |
+| yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms |
The following table lists additional changes to some configuration properties:
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 988177105dc..0415ae4e372 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -329,6 +329,9 @@ Release 2.8.0 - UNRELEASED
YARN-3800. Reduce storage footprint for ReservationAllocation. (Anubhav Dhoot
via curino)
+ YARN-3069. Document missing properties in yarn-default.xml.
+ (Ray Chiang via aajisaka)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4987084d29a..2edeef0b46e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -24,7 +24,8 @@
-
+
+
Factory to create client IPC classes.
yarn.ipc.client.factory.class
@@ -46,7 +47,8 @@
org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC
-
+
+
The hostname of the RM.
yarn.resourcemanager.hostname
@@ -142,6 +144,32 @@
${yarn.resourcemanager.hostname}:8090
+
+
+ The Kerberos keytab file to be used for spnego filter for the RM web
+ interface.
+
+ yarn.resourcemanager.webapp.spnego-keytab-file
+
+
+
+
+
+ The Kerberos principal to be used for spnego filter for the RM web
+ interface.
+
+ yarn.resourcemanager.webapp.spnego-principal
+
+
+
+
+
+ Add button to kill application in the RM Application view.
+
+ yarn.resourcemanager.webapp.ui-actions.enabled
+ true
+
+
yarn.resourcemanager.resource-tracker.address
${yarn.resourcemanager.hostname}:8031
@@ -280,7 +308,16 @@
- Enable RM to recover state after starting. If true, then
+
+ Used by node labels. If set to true, the port should be included in the
+ node name. Only usable if your scheduler supports node labels.
+
+ yarn.scheduler.include-port-in-node-name
+ false
+
+
+
+ Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified.
yarn.resourcemanager.recovery.enabled
false
@@ -672,6 +709,16 @@
+
+
+ The value specifies the file system (e.g. HDFS) path where ResourceManager
+ loads configuration if yarn.resourcemanager.configuration.provider-class
+ is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider.
+
+ yarn.resourcemanager.configuration.file-system-based-store
+ /yarn/conf
+
+
The setting that controls whether yarn system metrics is
published on the timeline server or not by RM.
@@ -695,6 +742,119 @@
+
+
+
+ RM DelegationTokenRenewer thread count
+
+ yarn.resourcemanager.delegation-token-renewer.thread-count
+ 50
+
+
+
+
+ RM secret key update interval in ms
+
+ yarn.resourcemanager.delegation.key.update-interval
+ 86400000
+
+
+
+
+ RM delegation token maximum lifetime in ms
+
+ yarn.resourcemanager.delegation.token.max-lifetime
+ 604800000
+
+
+
+
+ RM delegation token update interval in ms
+
+ yarn.resourcemanager.delegation.token.renew-interval
+ 86400000
+
+
+
+
+ Thread pool size for RMApplicationHistoryWriter.
+
+ yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size
+ 10
+
+
+
+
+ Comma-separated list of values (in minutes) for schedule queue related
+ metrics.
+
+ yarn.resourcemanager.metrics.runtime.buckets
+ 60,300,1440
+
+
+
+
+ Interval for the roll over for the master key used to generate
+ NodeManager tokens. It is expected to be set to a value much larger
+ than yarn.nm.liveness-monitor.expiry-interval-ms.
+
+ yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs
+ 86400
+
+
+
+
+ Flag to enable the ResourceManager reservation system.
+
+ yarn.resourcemanager.reservation-system.enable
+ false
+
+
+
+
+ The Java class to use as the ResourceManager reservation system.
+ By default, is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem
+ when using CapacityScheduler and is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem
+ when using FairScheduler.
+
+ yarn.resourcemanager.reservation-system.class
+
+
+
+
+
+ The plan follower policy class name to use for the ResourceManager
+ reservation system.
+ By default, is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower
+ is used when using CapacityScheduler, and is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower
+ when using FairScheduler.
+
+ yarn.resourcemanager.reservation-system.plan.follower
+
+
+
+
+
+ Step size of the reservation system in ms
+
+ yarn.resourcemanager.reservation-system.planfollower.time-step
+ 1000
+
+
+
+
+ The expiry interval for a container
+
+ yarn.resourcemanager.rm.container-allocation.expiry-interval-ms
+ 600000
+
+
+
+
The hostname of the NM.
yarn.nodemanager.hostname
@@ -727,7 +887,7 @@
Environment variables that containers may override rather than use NodeManager's default.
yarn.nodemanager.env-whitelist
- JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME
+ JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME
@@ -840,7 +1000,14 @@
- Where to store container logs. An application's localized log directory
+
+ yarn.nodemanager.container-localizer.java.opts
+ -Xmx256m
+
+
+
+
+ Where to store container logs. An application's localized log directory
will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
Individual containers' log directories will be below this, in directories
named container_{$contid}. Each container directory will contain the files
@@ -879,6 +1046,17 @@
-1
+
+
+ How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation
+ status is reported from NodeManager exceeds the configured value, RM
+ will report log aggregation status for this NodeManager as TIME_OUT
+
+ yarn.log-aggregation-status.time-out.ms
+ 600000
+
+
Time in seconds to retain user logs. Only applicable if
log aggregation is disabled
@@ -1005,6 +1183,32 @@
${yarn.nodemanager.hostname}:8042
+
+
+ The https adddress of the NM web application.
+
+ yarn.nodemanager.webapp.https.address
+ 0.0.0.0:8044
+
+
+
+
+ The Kerberos keytab file to be used for spnego filter for the NM web
+ interface.
+
+ yarn.nodemanager.webapp.spnego-keytab-file
+
+
+
+
+
+ The Kerberos principal to be used for spnego filter for the NM web
+ interface.
+
+ yarn.nodemanager.webapp.spnego-principal
+
+
+
How often to monitor containers.
yarn.nodemanager.container-monitor.interval-ms
@@ -1112,6 +1316,12 @@
yarn.nodemanager.linux-container-executor.cgroups.mount-path
+
+ Delay in ms between attempts to remove linux cgroup
+ yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms
+ 20
+
+
This determines which of the two modes that LCE should use on
a non-secure cluster. If this value is set to true, then all containers
@@ -1165,6 +1375,23 @@
false
+
+
+ Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container.
+
+ yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms
+ 1000
+
+
+
+
+ The UNIX group that the linux-container-executor should run as.
+
+ yarn.nodemanager.linux-container-executor.group
+
+
+
T-file compression types used to compress aggregated logs.
yarn.nodemanager.log-aggregation.compression-type
@@ -1272,6 +1499,66 @@
+
+
+ Adjustment to the container OS scheduling priority. In Linux, passed
+ directly to the nice command.
+
+ yarn.nodemanager.container-executor.os.sched.priority.adjustment
+ 0
+
+
+
+
+ Flag to enable container metrics
+
+ yarn.nodemanager.container-metrics.enable
+ true
+
+
+
+
+ Container metrics flush period in ms. Set to -1 for flush on completion.
+
+ yarn.nodemanager.container-metrics.period-ms
+ -1
+
+
+
+
+ Class used to calculate current container resource utilization.
+
+ yarn.nodemanager.container-monitor.process-tree.class
+
+
+
+
+
+ Flag to enable NodeManager disk health checker
+
+ yarn.nodemanager.disk-health-checker.enable
+ true
+
+
+
+
+ Number of threads to use in NM log cleanup. Used when log aggregation
+ is disabled.
+
+ yarn.nodemanager.log.deletion-threads-count
+ 4
+
+
+
+
+ The Windows group that the windows-container-executor should run as.
+
+ yarn.nodemanager.windows-secure-container-executor.group
+
+
+
+
+
yarn.nodemanager.docker-container-executor.exec-name
/usr/bin/docker
@@ -1280,24 +1567,28 @@
-
- yarn.nodemanager.aux-services.mapreduce_shuffle.class
- org.apache.hadoop.mapred.ShuffleHandler
+
+ The Docker image name to use for DockerContainerExecutor
+
+ yarn.nodemanager.docker-container-executor.image-name
+
-
- mapreduce.job.jar
-
-
+
mapreduce.job.hdfs-servers
${fs.defaultFS}
-
-
+
+ yarn.nodemanager.aux-services.mapreduce_shuffle.class
+ org.apache.hadoop.mapred.ShuffleHandler
+
+
+
+
The kerberos principal for the proxy, if the proxy is not
running as part of the RM.
@@ -1318,8 +1609,8 @@
-
-
+
+
CLASSPATH for YARN applications. A comma-separated list
@@ -1346,7 +1637,7 @@
-
+
Indicate to clients whether timeline service is enabled or not.
@@ -1530,6 +1821,7 @@
+
Whether the shared cache is enabled
yarn.sharedcache.enabled
@@ -1671,7 +1963,99 @@
20
-
+
+
+ ACL protocol for use in the Timeline server.
+
+ security.applicationhistory.protocol.acl
+
+
+
+
+
+
+
+ Set to true for MiniYARNCluster unit tests
+
+ yarn.is.minicluster
+ false
+
+
+
+
+ Set for MiniYARNCluster unit tests to control resource monitoring
+
+ yarn.minicluster.control-resource-monitoring
+ false
+
+
+
+
+ Set to false in order to allow MiniYARNCluster to run tests without
+ port conflicts.
+
+ yarn.minicluster.fixed.ports
+ false
+
+
+
+
+ Set to false in order to allow the NodeManager in MiniYARNCluster to
+ use RPC to talk to the RM.
+
+ yarn.minicluster.use-rpc
+ false
+
+
+
+
+ As yarn.nodemanager.resource.memory-mb property but for the NodeManager
+ in a MiniYARNCluster.
+
+ yarn.minicluster.yarn.nodemanager.resource.memory-mb
+ 4096
+
+
+
+
+
+
+ Enable node labels feature
+
+ yarn.node-labels.enabled
+ false
+
+
+
+
+ Retry policy used for FileSystem node label store. The policy is
+ specified by N pairs of sleep-time in milliseconds and number-of-retries
+ "s1,n1,s2,n2,...".
+
+ yarn.node-labels.fs-store.retry-policy-spec
+ 2000, 500
+
+
+
+
+ URI for NodeLabelManager. The default value is
+ /tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem.
+
+ yarn.node-labels.fs-store.root-dir
+
+
+
+
+
+ Set configuration type for node labels. Administrators can specify
+ "centralized" or "distributed".
+
+ yarn.node-labels.configuration-type
+ centralized
+
+
+
+
The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
@@ -1681,7 +2065,16 @@
- RSS usage of a process computed via
+
+ The duration (in ms) the YARN client waits for an expected state change
+ to occur. -1 means unlimited wait time.
+
+ yarn.client.application-client-protocol.poll-timeout-ms
+ -1
+
+
+
+ RSS usage of a process computed via
/proc/pid/stat is not very accurate as it includes shared pages of a
process. /proc/pid/smaps provides useful information like
Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used
@@ -1693,6 +2086,30 @@
false
+
+
+ URL for log aggregation server
+
+ yarn.log.server.url
+
+
+
+
+
+ RM Application Tracking URL
+
+ yarn.tracking.url.generator
+
+
+
+
+
+ Class to be used for YarnAuthorizationProvider
+
+ yarn.authorization-provider
+
+
+
Defines how often NMs wake up to upload log files.
The default value is -1. By default, the logs will be uploaded when
@@ -1703,4 +2120,15 @@
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds
-1
+
+
+
+ Enable/disable intermediate-data encryption at YARN level. For now,
+ this only is used by the FileSystemRMStateStore to setup right
+ file-system security attributes.
+
+ yarn.intermediate-data-encryption.enable
+ false
+
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 9075d9f6dd3..10e1ca6de66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -33,18 +33,19 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase;
*/
public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
+ @SuppressWarnings("deprecation")
@Override
public void initializeMemberVariables() {
xmlFilename = new String("yarn-default.xml");
configurationClasses = new Class[] { YarnConfiguration.class };
-
// Allocate for usage
configurationPropsToSkipCompare = new HashSet();
+ configurationPrefixToSkipCompare = new HashSet();
// Set error modes
errorIfMissingConfigProps = true;
- errorIfMissingXmlProps = false;
+ errorIfMissingXmlProps = true;
// Specific properties to skip
configurationPropsToSkipCompare
@@ -79,6 +80,37 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
+
+ // Ignore all YARN Application Timeline Service (version 1) properties
+ configurationPrefixToSkipCompare.add("yarn.timeline-service.");
+
+ // Used as Java command line properties, not XML
+ configurationPrefixToSkipCompare.add("yarn.app.container");
+
+ // Ignore NodeManager "work in progress" variables
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_INTERFACE);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED);
+
+ // Set by container-executor.cfg
+ configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR);
+
+ // Ignore deprecated properties
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
// Allocate for usage
xmlPropsToSkipCompare = new HashSet();
@@ -94,13 +126,11 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Used in the XML file as a variable reference internal to the XML file
xmlPropsToSkipCompare.add("yarn.nodemanager.hostname");
- xmlPropsToSkipCompare.add("yarn.timeline-service.hostname");
- // Currently defined in TimelineAuthenticationFilterInitializer
- xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication");
+ // Ignore all YARN Application Timeline Service (version 1) properties
+ xmlPrefixToSkipCompare.add("yarn.timeline-service");
- // Currently defined in RegistryConstants
+ // Currently defined in RegistryConstants/core-site.xml
xmlPrefixToSkipCompare.add("hadoop.registry");
}
-
}