From 20184e7ddc744076e7720d514b40fbebee55befd Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 3 Apr 2012 18:01:43 +0000 Subject: [PATCH] MAPREDUCE-4072. User set java.library.path seems to overwrite default creating problems native lib loading (Anupam Seth via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1309077 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 7 ++++++- .../src/site/apt/WritingYarnApplications.apt.vm | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a2b895bba5b..4c78a909a21 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -202,6 +202,9 @@ Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES + MAPREDUCE-4072. User set java.library.path seems to overwrite default + creating problems native lib loading (Anupam Seth via bobby) + NEW FEATURES IMPROVEMENTS diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index e668da3390e..a2bfa8080f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -412,7 +412,12 @@ -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. + maximum virtual memory of the child processes. + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm index 9048740ee95..7b72322fa14 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm @@ -779,6 +779,13 @@ Hadoop MapReduce Next Generation - Writing YARN Applications need to increase the value of the the cluster-wide configuration variable <<>>. +** How do I include native libraries? + + + Setting -Djava.library.path on the command line while launching a container + can cause native libraries used by Hadoop to not be loaded correctly and can + result in errors. It is cleaner to use LD_LIBRARY_PATH instead. + * Useful Links * {{{https://issues.apache.org/jira/secure/attachment/12486023/MapReduce_NextGen_Architecture.pdf}Map Reduce Next Generation Architecture}}