From ad6787aa4367b0de12b17fbc019098212e98a9f1 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 13 Jul 2015 12:57:01 +0900 Subject: [PATCH] YARN-3069. Document missing properties in yarn-default.xml. Contributed by Ray Chiang. (cherry picked from commit d6675606dc5f141c9af4f76a37128f8de4cfedad) --- .../src/site/markdown/DeprecatedProperties.md | 1 + hadoop-yarn-project/CHANGES.txt | 3 + .../src/main/resources/yarn-default.xml | 466 +++++++++++++++++- .../conf/TestYarnConfigurationFields.java | 44 +- 4 files changed, 488 insertions(+), 26 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index e9b82adc377..815ae5d5eca 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -276,6 +276,7 @@ The following table lists the configuration property names that are deprecated i | user.name | mapreduce.job.user.name | | webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted | | yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | +| yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms | The following table lists additional changes to some configuration properties: diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6e027d26d73..41f4a7963d2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -281,6 +281,9 @@ Release 2.8.0 - UNRELEASED YARN-3800. Reduce storage footprint for ReservationAllocation. (Anubhav Dhoot via curino) + YARN-3069. Document missing properties in yarn-default.xml. + (Ray Chiang via aajisaka) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4987084d29a..2edeef0b46e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -24,7 +24,8 @@ - + + Factory to create client IPC classes. yarn.ipc.client.factory.class @@ -46,7 +47,8 @@ org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC - + + The hostname of the RM. yarn.resourcemanager.hostname @@ -142,6 +144,32 @@ ${yarn.resourcemanager.hostname}:8090 + + + The Kerberos keytab file to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-principal + + + + + + Add button to kill application in the RM Application view. + + yarn.resourcemanager.webapp.ui-actions.enabled + true + + yarn.resourcemanager.resource-tracker.address ${yarn.resourcemanager.hostname}:8031 @@ -280,7 +308,16 @@ - Enable RM to recover state after starting. If true, then + + Used by node labels. If set to true, the port should be included in the + node name. Only usable if your scheduler supports node labels. + + yarn.scheduler.include-port-in-node-name + false + + + + Enable RM to recover state after starting. If true, then yarn.resourcemanager.store.class must be specified. yarn.resourcemanager.recovery.enabled false @@ -672,6 +709,16 @@ + + + The value specifies the file system (e.g. HDFS) path where ResourceManager + loads configuration if yarn.resourcemanager.configuration.provider-class + is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider. + + yarn.resourcemanager.configuration.file-system-based-store + /yarn/conf + + The setting that controls whether yarn system metrics is published on the timeline server or not by RM. @@ -695,6 +742,119 @@ + + + + RM DelegationTokenRenewer thread count + + yarn.resourcemanager.delegation-token-renewer.thread-count + 50 + + + + + RM secret key update interval in ms + + yarn.resourcemanager.delegation.key.update-interval + 86400000 + + + + + RM delegation token maximum lifetime in ms + + yarn.resourcemanager.delegation.token.max-lifetime + 604800000 + + + + + RM delegation token update interval in ms + + yarn.resourcemanager.delegation.token.renew-interval + 86400000 + + + + + Thread pool size for RMApplicationHistoryWriter. + + yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size + 10 + + + + + Comma-separated list of values (in minutes) for schedule queue related + metrics. + + yarn.resourcemanager.metrics.runtime.buckets + 60,300,1440 + + + + + Interval for the roll over for the master key used to generate + NodeManager tokens. It is expected to be set to a value much larger + than yarn.nm.liveness-monitor.expiry-interval-ms. + + yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs + 86400 + + + + + Flag to enable the ResourceManager reservation system. + + yarn.resourcemanager.reservation-system.enable + false + + + + + The Java class to use as the ResourceManager reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem + when using CapacityScheduler and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem + when using FairScheduler. + + yarn.resourcemanager.reservation-system.class + + + + + + The plan follower policy class name to use for the ResourceManager + reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower + is used when using CapacityScheduler, and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower + when using FairScheduler. + + yarn.resourcemanager.reservation-system.plan.follower + + + + + + Step size of the reservation system in ms + + yarn.resourcemanager.reservation-system.planfollower.time-step + 1000 + + + + + The expiry interval for a container + + yarn.resourcemanager.rm.container-allocation.expiry-interval-ms + 600000 + + + + The hostname of the NM. yarn.nodemanager.hostname @@ -727,7 +887,7 @@ Environment variables that containers may override rather than use NodeManager's default. yarn.nodemanager.env-whitelist - JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME @@ -840,7 +1000,14 @@ - Where to store container logs. An application's localized log directory + + yarn.nodemanager.container-localizer.java.opts + -Xmx256m + + + + + Where to store container logs. An application's localized log directory will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. Individual containers' log directories will be below this, in directories named container_{$contid}. Each container directory will contain the files @@ -879,6 +1046,17 @@ -1 + + + How long for ResourceManager to wait for NodeManager to report its + log aggregation status. If waiting time of which the log aggregation + status is reported from NodeManager exceeds the configured value, RM + will report log aggregation status for this NodeManager as TIME_OUT + + yarn.log-aggregation-status.time-out.ms + 600000 + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled @@ -1005,6 +1183,32 @@ ${yarn.nodemanager.hostname}:8042 + + + The https adddress of the NM web application. + + yarn.nodemanager.webapp.https.address + 0.0.0.0:8044 + + + + + The Kerberos keytab file to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-principal + + + How often to monitor containers. yarn.nodemanager.container-monitor.interval-ms @@ -1112,6 +1316,12 @@ yarn.nodemanager.linux-container-executor.cgroups.mount-path + + Delay in ms between attempts to remove linux cgroup + yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms + 20 + + This determines which of the two modes that LCE should use on a non-secure cluster. If this value is set to true, then all containers @@ -1165,6 +1375,23 @@ false + + + Interval of time the linux container executor should try cleaning up + cgroups entry when cleaning up a container. + + yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms + 1000 + + + + + The UNIX group that the linux-container-executor should run as. + + yarn.nodemanager.linux-container-executor.group + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type @@ -1272,6 +1499,66 @@ + + + Adjustment to the container OS scheduling priority. In Linux, passed + directly to the nice command. + + yarn.nodemanager.container-executor.os.sched.priority.adjustment + 0 + + + + + Flag to enable container metrics + + yarn.nodemanager.container-metrics.enable + true + + + + + Container metrics flush period in ms. Set to -1 for flush on completion. + + yarn.nodemanager.container-metrics.period-ms + -1 + + + + + Class used to calculate current container resource utilization. + + yarn.nodemanager.container-monitor.process-tree.class + + + + + + Flag to enable NodeManager disk health checker + + yarn.nodemanager.disk-health-checker.enable + true + + + + + Number of threads to use in NM log cleanup. Used when log aggregation + is disabled. + + yarn.nodemanager.log.deletion-threads-count + 4 + + + + + The Windows group that the windows-container-executor should run as. + + yarn.nodemanager.windows-secure-container-executor.group + + + + + yarn.nodemanager.docker-container-executor.exec-name /usr/bin/docker @@ -1280,24 +1567,28 @@ - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler + + The Docker image name to use for DockerContainerExecutor + + yarn.nodemanager.docker-container-executor.image-name + - - mapreduce.job.jar - - + mapreduce.job.hdfs-servers ${fs.defaultFS} - - + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + The kerberos principal for the proxy, if the proxy is not running as part of the RM. @@ -1318,8 +1609,8 @@ - - + + CLASSPATH for YARN applications. A comma-separated list @@ -1346,7 +1637,7 @@ - + Indicate to clients whether timeline service is enabled or not. @@ -1530,6 +1821,7 @@ + Whether the shared cache is enabled yarn.sharedcache.enabled @@ -1671,7 +1963,99 @@ 20 - + + + ACL protocol for use in the Timeline server. + + security.applicationhistory.protocol.acl + + + + + + + + Set to true for MiniYARNCluster unit tests + + yarn.is.minicluster + false + + + + + Set for MiniYARNCluster unit tests to control resource monitoring + + yarn.minicluster.control-resource-monitoring + false + + + + + Set to false in order to allow MiniYARNCluster to run tests without + port conflicts. + + yarn.minicluster.fixed.ports + false + + + + + Set to false in order to allow the NodeManager in MiniYARNCluster to + use RPC to talk to the RM. + + yarn.minicluster.use-rpc + false + + + + + As yarn.nodemanager.resource.memory-mb property but for the NodeManager + in a MiniYARNCluster. + + yarn.minicluster.yarn.nodemanager.resource.memory-mb + 4096 + + + + + + + Enable node labels feature + + yarn.node-labels.enabled + false + + + + + Retry policy used for FileSystem node label store. The policy is + specified by N pairs of sleep-time in milliseconds and number-of-retries + "s1,n1,s2,n2,...". + + yarn.node-labels.fs-store.retry-policy-spec + 2000, 500 + + + + + URI for NodeLabelManager. The default value is + /tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem. + + yarn.node-labels.fs-store.root-dir + + + + + + Set configuration type for node labels. Administrators can specify + "centralized" or "distributed". + + yarn.node-labels.configuration-type + centralized + + + + The interval that the yarn client library uses to poll the completion status of the asynchronous API of application client protocol. @@ -1681,7 +2065,16 @@ - RSS usage of a process computed via + + The duration (in ms) the YARN client waits for an expected state change + to occur. -1 means unlimited wait time. + + yarn.client.application-client-protocol.poll-timeout-ms + -1 + + + + RSS usage of a process computed via /proc/pid/stat is not very accurate as it includes shared pages of a process. /proc/pid/smaps provides useful information like Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used @@ -1693,6 +2086,30 @@ false + + + URL for log aggregation server + + yarn.log.server.url + + + + + + RM Application Tracking URL + + yarn.tracking.url.generator + + + + + + Class to be used for YarnAuthorizationProvider + + yarn.authorization-provider + + + Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded when @@ -1703,4 +2120,15 @@ yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds -1 + + + + Enable/disable intermediate-data encryption at YARN level. For now, + this only is used by the FileSystemRMStateStore to setup right + file-system security attributes. + + yarn.intermediate-data-encryption.enable + false + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9075d9f6dd3..10e1ca6de66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -33,18 +33,19 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase; */ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { + @SuppressWarnings("deprecation") @Override public void initializeMemberVariables() { xmlFilename = new String("yarn-default.xml"); configurationClasses = new Class[] { YarnConfiguration.class }; - // Allocate for usage configurationPropsToSkipCompare = new HashSet(); + configurationPrefixToSkipCompare = new HashSet(); // Set error modes errorIfMissingConfigProps = true; - errorIfMissingXmlProps = false; + errorIfMissingXmlProps = true; // Specific properties to skip configurationPropsToSkipCompare @@ -79,6 +80,37 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { configurationPropsToSkipCompare .add(YarnConfiguration .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL); + + // Ignore all YARN Application Timeline Service (version 1) properties + configurationPrefixToSkipCompare.add("yarn.timeline-service."); + + // Used as Java command line properties, not XML + configurationPrefixToSkipCompare.add("yarn.app.container"); + + // Ignore NodeManager "work in progress" variables + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_INTERFACE); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED); + + // Set by container-executor.cfg + configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); + + // Ignore deprecated properties + configurationPrefixToSkipCompare + .add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS); // Allocate for usage xmlPropsToSkipCompare = new HashSet(); @@ -94,13 +126,11 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { // Used in the XML file as a variable reference internal to the XML file xmlPropsToSkipCompare.add("yarn.nodemanager.hostname"); - xmlPropsToSkipCompare.add("yarn.timeline-service.hostname"); - // Currently defined in TimelineAuthenticationFilterInitializer - xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication"); + // Ignore all YARN Application Timeline Service (version 1) properties + xmlPrefixToSkipCompare.add("yarn.timeline-service"); - // Currently defined in RegistryConstants + // Currently defined in RegistryConstants/core-site.xml xmlPrefixToSkipCompare.add("hadoop.registry"); } - }