Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-11464. Reinstate support for launching Hadoop processes on Windows using Cygwin. Contributed by Chris Nauroth.

This commit is contained in:
cnauroth 2015-01-09 15:01:33 -08:00
parent a663faf741
commit 9a7c763b3f
6 changed files with 104 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -106,6 +106,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11032. Replace use of Guava's Stopwatch with Hadoop's StopWatch
(ozawa)
HADOOP-11464. Reinstate support for launching Hadoop processes on Windows
using Cygwin. (cnauroth)
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.

22
hadoop-common-project/hadoop-common/src/main/bin/hadoop
View File

@ -22,6 +22,12 @@ bin=`dirname ${bin}`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
cygwin=false
case "$(uname)" in
CYGWIN*) cygwin=true;;
esac
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
@ -125,6 +131,9 @@ case $COMMAND in
elif [ "$COMMAND" = "classpath" ] ; then
if [ "$#" -eq 1 ]; then
# No need to bother starting up a JVM for this simple case.
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
fi
echo $CLASSPATH
exit
else
@ -137,6 +146,19 @@ case $COMMAND in
else
CLASS=$COMMAND
fi
# cygwin path translation
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
HADOOP_LOG_DIR=$(cygpath -w "$HADOOP_LOG_DIR" 2>/dev/null)
HADOOP_PREFIX=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
HADOOP_CONF_DIR=$(cygpath -w "$HADOOP_CONF_DIR" 2>/dev/null)
HADOOP_COMMON_HOME=$(cygpath -w "$HADOOP_COMMON_HOME" 2>/dev/null)
HADOOP_HDFS_HOME=$(cygpath -w "$HADOOP_HDFS_HOME" 2>/dev/null)
HADOOP_YARN_HOME=$(cygpath -w "$HADOOP_YARN_HOME" 2>/dev/null)
HADOOP_MAPRED_HOME=$(cygpath -w "$HADOOP_MAPRED_HOME" 2>/dev/null)
fi
shift
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS

10
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
View File

@ -245,10 +245,18 @@ TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
HADOOP_HOME=$HADOOP_PREFIX
if $cygwin; then
HADOOP_HOME=$(cygpath -w "$HADOOP_HOME" 2>/dev/null)
fi
export HADOOP_HOME
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_HOME"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
if $cygwin; then
JAVA_LIBRARY_PATH=$(cygpath -w "$JAVA_LIBRARY_PATH" 2>/dev/null)
fi
HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH
fi

18
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
View File

@ -30,6 +30,12 @@ bin=`dirname ${bin}`
bin=`cd "$bin" > /dev/null; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
cygwin=false
case "$(uname)" in
CYGWIN*) cygwin=true;;
esac
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
@ -205,6 +211,18 @@ else
CLASS="$COMMAND"
fi
# cygwin path translation
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
HADOOP_LOG_DIR=$(cygpath -w "$HADOOP_LOG_DIR" 2>/dev/null)
HADOOP_PREFIX=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
HADOOP_CONF_DIR=$(cygpath -w "$HADOOP_CONF_DIR" 2>/dev/null)
HADOOP_COMMON_HOME=$(cygpath -w "$HADOOP_COMMON_HOME" 2>/dev/null)
HADOOP_HDFS_HOME=$(cygpath -w "$HADOOP_HDFS_HOME" 2>/dev/null)
HADOOP_YARN_HOME=$(cygpath -w "$HADOOP_YARN_HOME" 2>/dev/null)
HADOOP_MAPRED_HOME=$(cygpath -w "$HADOOP_MAPRED_HOME" 2>/dev/null)
fi
export CLASSPATH=$CLASSPATH
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"

21
hadoop-mapreduce-project/bin/mapred
View File

@ -20,6 +20,12 @@ bin=`dirname ${bin}`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
cygwin=false
case "$(uname)" in
CYGWIN*) cygwin=true;;
esac
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
if [ -e ${HADOOP_LIBEXEC_DIR}/mapred-config.sh ]; then
. ${HADOOP_LIBEXEC_DIR}/mapred-config.sh
@ -142,10 +148,25 @@ for f in $HADOOP_MAPRED_HOME/modules/*.jar; do
done
if [ "$COMMAND" = "classpath" ] ; then
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
fi
echo $CLASSPATH
exit
fi
# cygwin path translation
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
HADOOP_LOG_DIR=$(cygpath -w "$HADOOP_LOG_DIR" 2>/dev/null)
HADOOP_PREFIX=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
HADOOP_CONF_DIR=$(cygpath -w "$HADOOP_CONF_DIR" 2>/dev/null)
HADOOP_COMMON_HOME=$(cygpath -w "$HADOOP_COMMON_HOME" 2>/dev/null)
HADOOP_HDFS_HOME=$(cygpath -w "$HADOOP_HDFS_HOME" 2>/dev/null)
HADOOP_YARN_HOME=$(cygpath -w "$HADOOP_YARN_HOME" 2>/dev/null)
HADOOP_MAPRED_HOME=$(cygpath -w "$HADOOP_MAPRED_HOME" 2>/dev/null)
fi
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
export CLASSPATH

32
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -55,6 +55,12 @@ bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
cygwin=false
case "$(uname)" in
CYGWIN*) cygwin=true;;
esac
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
@ -202,6 +208,9 @@ unset IFS
# figure out which class to run
if [ "$COMMAND" = "classpath" ] ; then
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
fi
echo $CLASSPATH
exit
elif [ "$COMMAND" = "rmadmin" ] ; then
@ -281,15 +290,36 @@ else
CLASS=$COMMAND
fi
# cygwin path translation
if $cygwin; then
CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
HADOOP_LOG_DIR=$(cygpath -w "$HADOOP_LOG_DIR" 2>/dev/null)
HADOOP_PREFIX=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
HADOOP_CONF_DIR=$(cygpath -w "$HADOOP_CONF_DIR" 2>/dev/null)
HADOOP_COMMON_HOME=$(cygpath -w "$HADOOP_COMMON_HOME" 2>/dev/null)
HADOOP_HDFS_HOME=$(cygpath -w "$HADOOP_HDFS_HOME" 2>/dev/null)
HADOOP_YARN_HOME=$(cygpath -w "$HADOOP_YARN_HOME" 2>/dev/null)
HADOOP_MAPRED_HOME=$(cygpath -w "$HADOOP_MAPRED_HOME" 2>/dev/null)
YARN_LOG_DIR=$(cygpath -w "$YARN_LOG_DIR" 2>/dev/null)
fi
YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_YARN_HOME"
HADOOP_HOME=$HADOOP_PREFIX
if $cygwin; then
HADOOP_HOME=$(cygpath -w "$HADOOP_HOME" 2>/dev/null)
fi
export HADOOP_HOME
YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_HOME"
YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
if $cygwin; then
JAVA_LIBRARY_PATH=$(cygpath -w "$JAVA_LIBRARY_PATH" 2>/dev/null)
fi
YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi