diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8113cfba200..aaa898f5f9e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -148,6 +148,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6574. MR AM should print host of failed tasks. (Mohammad Shahid Khan via wangda) + MAPREDUCE-6584. Remove trailing whitespaces from mapred-default.xml + (aajisaka) + OPTIMIZATIONS MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index ae84f8c6cf2..1e150a985e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -27,7 +27,7 @@ mapreduce.job.committer.setup.cleanup.needed true true, if job needs job-setup and job-cleanup. - false, otherwise + false, otherwise @@ -42,7 +42,7 @@ mapreduce.task.io.sort.mb 100 - The total amount of buffer memory to use while sorting + The total amount of buffer memory to use while sorting files, in megabytes. By default, gives each merge stream 1MB, which should minimize seeks. @@ -76,7 +76,7 @@ mapreduce.job.reduces 1 The default number of reduce tasks per job. Typically set to 99% - of the cluster's reduce capacity, so that if a node fails the reduces can + of the cluster's reduce capacity, so that if a node fails the reduces can still be executed in a single wave. Ignored when mapreduce.framework.name is "local". @@ -122,7 +122,7 @@ mapreduce.job.max.split.locations 10 - The max number of block locations to store for each split for + The max number of block locations to store for each split for locality calculation. @@ -130,7 +130,7 @@ mapreduce.job.split.metainfo.maxsize 10000000 - The maximum permissible size of the split metainfo file. + The maximum permissible size of the split metainfo file. The MapReduce ApplicationMaster won't attempt to read submitted split metainfo files bigger than this configured value. No limits if set to -1. @@ -164,7 +164,7 @@ mapreduce.reduce.shuffle.fetch.retry.interval-ms 1000 - Time of interval that fetcher retry to fetch again when some + Time of interval that fetcher retry to fetch again when some non-fatal failure happens because of some events like NM restart. @@ -172,7 +172,7 @@ mapreduce.reduce.shuffle.fetch.retry.timeout-ms 30000 - Timeout value for fetcher to retry to fetch again when some + Timeout value for fetcher to retry to fetch again when some non-fatal failure happens because of some events like NM restart. @@ -269,16 +269,16 @@ mapred.child.java.opts -Xmx200m Java opts for the task processes. - The following symbol, if present, will be interpolated: @taskid@ is replaced + The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc Usage of -Djava.library.path can cause programs to no longer function if - hadoop native libraries are used. These values should instead be set as part - of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and - mapreduce.reduce.env config settings. + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. @@ -335,12 +335,12 @@ mapreduce.admin.user.env - Expert: Additional execution environment entries for + Expert: Additional execution environment entries for map and reduce task processes. This is not an additive property. You must preserve the original value if you want your map and - reduce tasks to have access to native libraries (compression, etc). - When this value is empty, the command to set execution - envrionment will be OS dependent: + reduce tasks to have access to native libraries (compression, etc). + When this value is empty, the command to set execution + envrionment will be OS dependent: For linux, use LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native. For windows, use PATH = %PATH%;%HADOOP_COMMON_HOME%\\bin. @@ -385,7 +385,7 @@ mapreduce.reduce.merge.inmem.threshold 1000 - The threshold, in terms of the number of files + The threshold, in terms of the number of files for the in-memory merge process. When we accumulate threshold number of files we initiate the in-memory merge and spill to disk. A value of 0 or less than 0 indicates we want to DON'T have any threshold and instead depend only on @@ -464,12 +464,12 @@ mapreduce.shuffle.transferTo.allowed - This option can enable/disable using nio transferTo method in - the shuffle phase. NIO transferTo does not perform well on windows in the - shuffle phase. Thus, with this configuration property it is possible to - disable it, in which case custom transfer method will be used. Recommended - value is false when running Hadoop on Windows. For Linux, it is recommended - to set it to true. If nothing is set then the default value is false for + This option can enable/disable using nio transferTo method in + the shuffle phase. NIO transferTo does not perform well on windows in the + shuffle phase. Thus, with this configuration property it is possible to + disable it, in which case custom transfer method will be used. Recommended + value is false when running Hadoop on Windows. For Linux, it is recommended + to set it to true. If nothing is set then the default value is false for Windows, and true for Linux. @@ -477,8 +477,8 @@ mapreduce.shuffle.transfer.buffer.size 131072 - This property is used only if - mapreduce.shuffle.transferTo.allowed is set to false. In that case, + This property is used only if + mapreduce.shuffle.transferTo.allowed is set to false. In that case, this property defines the size of the buffer used in the buffer copy code for the shuffle phase. The size of this buffer determines the size of the IO requests. @@ -496,14 +496,14 @@ mapreduce.map.speculative true - If true, then multiple instances of some map tasks + If true, then multiple instances of some map tasks may be executed in parallel. mapreduce.reduce.speculative true - If true, then multiple instances of some reduce tasks + If true, then multiple instances of some reduce tasks may be executed in parallel. @@ -551,7 +551,7 @@ of the collectors in turn. The first to successfully initialize will be used. - + mapreduce.job.speculative.slowtaskthreshold 1.0 @@ -650,14 +650,14 @@ mapreduce.task.files.preserve.failedtasks false - Should the files for failed tasks be kept. This should only be + Should the files for failed tasks be kept. This should only be used on jobs that are failing, because the storage is never reclaimed. It also prevents the map outputs from being erased from the reduce directory as they are consumed. - mapreduce.job.queuename @@ -967,7 +967,7 @@ Queue to which a job is submitted. This must match one of the queues defined in mapred-queues.xml for the system. Also, the ACL setup for the queue must allow the current user to submit a job to the queue. - Before specifying a queue, ensure that the system is configured with + Before specifying a queue, ensure that the system is configured with the queue, and access is allowed for submitting jobs to the queue. @@ -975,7 +975,7 @@ mapreduce.job.tags - Tags for the job that will be passed to YARN at submission + Tags for the job that will be passed to YARN at submission time. Queries to YARN for applications can filter on these tags. @@ -1110,15 +1110,15 @@ mapreduce.job.reduce.slowstart.completedmaps 0.05 - Fraction of the number of maps in the job which should be - complete before reduces are scheduled for the job. + Fraction of the number of maps in the job which should be + complete before reduces are scheduled for the job. mapreduce.job.complete.cancel.delegation.tokens true - if false - do not unregister/cancel delegation tokens from + if false - do not unregister/cancel delegation tokens from renewal, because same tokens may be used by spawned jobs @@ -1126,8 +1126,8 @@ mapreduce.shuffle.port 13562 - Default port that the ShuffleHandler will run on. ShuffleHandler - is a service run at the NodeManager to facilitate transfers of intermediate + Default port that the ShuffleHandler will run on. ShuffleHandler + is a service run at the NodeManager to facilitate transfers of intermediate Map outputs to requesting Reducers. @@ -1273,38 +1273,38 @@ yarn.app.mapreduce.am.env - User added environment variables for the MR App Master + User added environment variables for the MR App Master processes. Example : 1) A=foo This will set the env variable A to foo - 2) B=$B:c This is inherit tasktracker's B env variable. + 2) B=$B:c This is inherit tasktracker's B env variable. yarn.app.mapreduce.am.admin.user.env - Environment variables for the MR App Master - processes for admin purposes. These values are set first and can be + Environment variables for the MR App Master + processes for admin purposes. These values are set first and can be overridden by the user env (yarn.app.mapreduce.am.env) Example : 1) A=foo This will set the env variable A to foo - 2) B=$B:c This is inherit app master's B env variable. + 2) B=$B:c This is inherit app master's B env variable. yarn.app.mapreduce.am.command-opts -Xmx1024m - Java opts for the MR App Master processes. - The following symbol, if present, will be interpolated: @taskid@ is replaced + Java opts for the MR App Master processes. + The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - + Usage of -Djava.library.path can cause programs to no longer function if - hadoop native libraries are used. These values should instead be set as part - of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and - mapreduce.reduce.env config settings. + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. @@ -1313,19 +1313,19 @@ Java opts for the MR App Master processes for admin purposes. It will appears before the opts set by yarn.app.mapreduce.am.command-opts and - thus its options can be overridden user. - + thus its options can be overridden user. + Usage of -Djava.library.path can cause programs to no longer function if - hadoop native libraries are used. These values should instead be set as part - of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and - mapreduce.reduce.env config settings. + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. yarn.app.mapreduce.am.job.task.listener.thread-count 30 - The number of threads used to handle RPC calls in the + The number of threads used to handle RPC calls in the MR AppMaster from remote tasks @@ -1333,7 +1333,7 @@ yarn.app.mapreduce.am.job.client.port-range Range of ports that the MapReduce AM can use when binding. - Leave blank if you want all possible ports. + Leave blank if you want all possible ports. For example 50000-50050,50100-50200 @@ -1617,7 +1617,7 @@ mapreduce.jobhistory.cleaner.interval-ms 86400000 - How often the job history cleaner checks for files to delete, + How often the job history cleaner checks for files to delete, in milliseconds. Defaults to 86400000 (one day). Files are only deleted if they are older than mapreduce.jobhistory.max-age-ms.