From e76f7b73308ddd4fce982d948aa30be3624f274a Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 3 Apr 2012 18:03:18 +0000 Subject: [PATCH] svn merge -c 1309077 from trunk. FIXES MAPREDUCE-4072. User set java.library.path seems to overwrite default creating problems native lib loading (Anupam Seth via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1309079 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 7 ++++++- .../src/site/apt/WritingYarnApplications.apt.vm | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4b7c4cbaf3f..e44822831a4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -103,6 +103,9 @@ Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES + MAPREDUCE-4072. User set java.library.path seems to overwrite default + creating problems native lib loading (Anupam Seth via bobby) + NEW FEATURES IMPROVEMENTS diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index e668da3390e..a2bfa8080f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -412,7 +412,12 @@ -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. + maximum virtual memory of the child processes. + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm index 9048740ee95..7b72322fa14 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm @@ -779,6 +779,13 @@ Hadoop MapReduce Next Generation - Writing YARN Applications need to increase the value of the the cluster-wide configuration variable <<>>. +** How do I include native libraries? + + + Setting -Djava.library.path on the command line while launching a container + can cause native libraries used by Hadoop to not be loaded correctly and can + result in errors. It is cleaner to use LD_LIBRARY_PATH instead. + * Useful Links * {{{https://issues.apache.org/jira/secure/attachment/12486023/MapReduce_NextGen_Architecture.pdf}Map Reduce Next Generation Architecture}}